{
 "cells": [
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Imports and chains"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "from langchain.llms import OpenAI \n",
    "from langchain.prompts import PromptTemplate \n",
    "from langchain.chains.llm import LLMChain\n",
    "from cot import Collection\n",
    "import json\n",
    "from langchain.chat_models import ChatOpenAI\n",
    "# from dataloader import to_Collection"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "\"\"\"CoT Chain\"\"\"\n",
    "\n",
    "llm = ChatOpenAI(temperature=.0,model_name=\"gpt-3.5-turbo\") #ADA #for chat: gpt-3.5-turbo\n",
    "#llm = ChatOpenAI(temperature=.0,model_name=\"gpt-4\") \n",
    "#llm = OpenAI(temperature=.0,model_name=\"text-davinci-003\") \n",
    "\"\"\"answer extraction chain\"\"\"\n",
    "\n",
    "extraction_template = \"\"\"{instruction}\n",
    "\n",
    "Question: {question}\n",
    "Answer_choices: {answer_choices}\n",
    "\n",
    "Cot: {cot_trigger}{cot}\n",
    "{answer_extraction}\n",
    "\"\"\"\n",
    "\n",
    "prompt_template = PromptTemplate(input_variables=[\"instruction\",\"question\",\"answer_choices\",\"cot_trigger\",\"cot\",\"answer_extraction\"], template=extraction_template)\n",
    "answer_chain = LLMChain(llm=llm, prompt=prompt_template,output_key=\"predicted_answer\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "reflect_template = \"\"\"\n",
    "    Question: {question}\n",
    "    Answer_choices: {answer_choices}\n",
    "\n",
    "    {cot_trigger} {cot}\n",
    "    {answer_extraction} {answer}\n",
    "    \n",
    "    Reflection: {reflection_prompt}\n",
    "    \"\"\"\n",
    "reflect_prompt_template = PromptTemplate(input_variables=[\"question\",\"answer_choices\",\"cot_trigger\",\"cot\",\"answer_extraction\",'answer','reflection_prompt'], template=reflect_template)\n",
    "reflect_chain = LLMChain(llm=llm, prompt=reflect_prompt_template,output_key=\"reflection\")\n",
    "\n",
    "#{instruction}\n",
    "extraction_template = \"\"\"\n",
    "\n",
    "    Question: {question}\n",
    "    Answer_choices: {answer_choices}\n",
    "\n",
    "    {cot_trigger} {cot}\n",
    "    {answer_extraction} {answer}\n",
    "    \n",
    "    Reflection: {reflection_prompt}\n",
    "    {reflection}\n",
    "\n",
    "    {reflect_answer_extraction}\n",
    "    \"\"\"\n",
    "    #Get reflection\n",
    "ans_prompt_template = PromptTemplate(input_variables=[\"question\",\"answer_choices\",\"cot_trigger\",\"cot\",\"answer_extraction\",'answer','reflection_prompt','reflection','reflect_answer_extraction'], template=extraction_template)\n",
    "reflect_answer_chain = LLMChain(llm=llm, prompt=ans_prompt_template,output_key=\"reflection_answer\")\n",
    "\n",
    "    # This is the overall chain where we run these two chains in sequence.\n",
    "from langchain.chains import SequentialChain\n",
    "reflect_overall_chain = SequentialChain(chains=[reflect_chain, reflect_answer_chain],input_variables=[\"question\",\"answer_choices\",\"cot_trigger\",\"answer_extraction\",'cot','answer','reflection_prompt','reflect_answer_extraction'],\n",
    "        output_variables=[\"reflection\", \"reflection_answer\"],\n",
    "        verbose=True)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\"\"\"Extract script: Assumes there are CoTs in the dataset already\"\"\"\n",
    "ts_hard = Collection.from_json('ts_hard_v1.json')\n",
    "ts_hard.unload_datasets([\"med_qa\",\"medmc_qa\",\"commonsense_qa\",\"open_book_qa\",\"worldtree\"])\n",
    "\n",
    "\n",
    "input_dict = {\n",
    "    \"instruction\": \"\",\n",
    "    \"cot_trigger\": \"Answer: Let's think step by step.\",\n",
    "    \"answer_extraction\": \"The question was created by crowdsourcing with the goal to test AI systems. Multiple answers may be valid but only one answer will be regarded as correct. If you give more than one answer, it will be evaluated as wrong. Pick the one answer that is most likely to be correct for this question. Therefore, among A through D, the answer is\" \n",
    "}\n",
    "\n",
    "extract = ts_hard.extract_flexible(chain=answer_chain,input_dict=input_dict)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "3bac7f2e14be4a8d87854f97922090d9",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/11 [00:00<?, ?ex/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"font-weight: bold\">{</span>\n",
       "<span style=\"color: #7fbf7f; text-decoration-color: #7fbf7f\">│   </span><span style=\"color: #008000; text-decoration-color: #008000\">'strategy_qa'</span>: <span style=\"font-weight: bold\">{</span>\n",
       "<span style=\"color: #7fbf7f; text-decoration-color: #7fbf7f\">│   │   </span><span style=\"color: #008000; text-decoration-color: #008000\">'train'</span>: <span style=\"font-weight: bold\">{</span>\n",
       "<span style=\"color: #7fbf7f; text-decoration-color: #7fbf7f\">│   │   │   </span><span style=\"color: #008000; text-decoration-color: #008000\">'accuracy'</span>: <span style=\"font-weight: bold\">{</span>\n",
       "<span style=\"color: #7fbf7f; text-decoration-color: #7fbf7f\">│   │   │   │   </span><span style=\"color: #008000; text-decoration-color: #008000\">'text-davinci-003'</span>: <span style=\"font-weight: bold\">{</span>\n",
       "<span style=\"color: #7fbf7f; text-decoration-color: #7fbf7f\">│   │   │   │   │   </span><span style=\"color: #008000; text-decoration-color: #008000\">'None_kojima-01_The question was created by crowdsourcing with the goal to test AI systems. Multiple answers may be valid but only one answer will be regarded as correct. If you give more than one answer, it will be evaluated as wrong. Pick the one answer that is most likely to be correct for this question. Therefore, among A through D, the answer is'</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0.181818</span>,\n",
       "<span style=\"color: #7fbf7f; text-decoration-color: #7fbf7f\">│   │   │   │   │   </span><span style=\"color: #008000; text-decoration-color: #008000\">'None_kojima-01_kojima-yes-no'</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0.0</span>\n",
       "<span style=\"color: #7fbf7f; text-decoration-color: #7fbf7f\">│   │   │   │   </span><span style=\"font-weight: bold\">}</span>\n",
       "<span style=\"color: #7fbf7f; text-decoration-color: #7fbf7f\">│   │   │   </span><span style=\"font-weight: bold\">}</span>\n",
       "<span style=\"color: #7fbf7f; text-decoration-color: #7fbf7f\">│   │   </span><span style=\"font-weight: bold\">}</span>\n",
       "<span style=\"color: #7fbf7f; text-decoration-color: #7fbf7f\">│   </span><span style=\"font-weight: bold\">}</span>\n",
       "<span style=\"font-weight: bold\">}</span>\n",
       "</pre>\n"
      ],
      "text/plain": [
       "\u001b[1m{\u001b[0m\n",
       "\u001b[2;32m│   \u001b[0m\u001b[32m'strategy_qa'\u001b[0m: \u001b[1m{\u001b[0m\n",
       "\u001b[2;32m│   │   \u001b[0m\u001b[32m'train'\u001b[0m: \u001b[1m{\u001b[0m\n",
       "\u001b[2;32m│   │   │   \u001b[0m\u001b[32m'accuracy'\u001b[0m: \u001b[1m{\u001b[0m\n",
       "\u001b[2;32m│   │   │   │   \u001b[0m\u001b[32m'text-davinci-003'\u001b[0m: \u001b[1m{\u001b[0m\n",
       "\u001b[2;32m│   │   │   │   │   \u001b[0m\u001b[32m'None_kojima-01_The question was created by crowdsourcing with the goal to test AI systems. Multiple answers may be valid but only one answer will be regarded as correct. If you give more than one answer, it will be evaluated as wrong. Pick the one answer that is most likely to be correct for this question. Therefore, among A through D, the answer is'\u001b[0m: \u001b[1;36m0.181818\u001b[0m,\n",
       "\u001b[2;32m│   │   │   │   │   \u001b[0m\u001b[32m'None_kojima-01_kojima-yes-no'\u001b[0m: \u001b[1;36m0.0\u001b[0m\n",
       "\u001b[2;32m│   │   │   │   \u001b[0m\u001b[1m}\u001b[0m\n",
       "\u001b[2;32m│   │   │   \u001b[0m\u001b[1m}\u001b[0m\n",
       "\u001b[2;32m│   │   \u001b[0m\u001b[1m}\u001b[0m\n",
       "\u001b[2;32m│   \u001b[0m\u001b[1m}\u001b[0m\n",
       "\u001b[1m}\u001b[0m\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "test = Collection.to_Collection(extract,\"strategy_qa\",'train','file_test')\n",
    "eval = test.evaluate()\n",
    "from rich.pretty import pprint\n",
    "pprint(eval)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "ts_hard = Collection.from_json('ts_hard_v1.json')\n",
    "ts_hard.unload_datasets([\"med_qa\",\"medmc_qa\",\"commonsense_qa\",\"strategy_qa\",\"worldtree\"])\n",
    "extract = ts_hard.extract_flexible(chain=answer_chain,input_dict=input_dict)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "80860638e62e4d78be0d05dcfb83de68",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/21 [00:00<?, ?ex/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"font-weight: bold\">{</span>\n",
       "<span style=\"color: #7fbf7f; text-decoration-color: #7fbf7f\">│   </span><span style=\"color: #008000; text-decoration-color: #008000\">'open_book_qa'</span>: <span style=\"font-weight: bold\">{</span>\n",
       "<span style=\"color: #7fbf7f; text-decoration-color: #7fbf7f\">│   │   </span><span style=\"color: #008000; text-decoration-color: #008000\">'test'</span>: <span style=\"font-weight: bold\">{</span>\n",
       "<span style=\"color: #7fbf7f; text-decoration-color: #7fbf7f\">│   │   │   </span><span style=\"color: #008000; text-decoration-color: #008000\">'accuracy'</span>: <span style=\"font-weight: bold\">{</span>\n",
       "<span style=\"color: #7fbf7f; text-decoration-color: #7fbf7f\">│   │   │   │   </span><span style=\"color: #008000; text-decoration-color: #008000\">'text-davinci-003'</span>: <span style=\"font-weight: bold\">{</span>\n",
       "<span style=\"color: #7fbf7f; text-decoration-color: #7fbf7f\">│   │   │   │   │   </span><span style=\"color: #008000; text-decoration-color: #008000\">'None_kojima-01_The question was created by crowdsourcing with the goal to test AI systems. Multiple answers may be valid but only one answer will be regarded as correct. If you give more than one answer, it will be evaluated as wrong. Pick the one answer that is most likely to be correct for this question. Therefore, among A through D, the answer is'</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0.047619</span>,\n",
       "<span style=\"color: #7fbf7f; text-decoration-color: #7fbf7f\">│   │   │   │   │   </span><span style=\"color: #008000; text-decoration-color: #008000\">'None_kojima-01_kojima-A-D'</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0.0</span>\n",
       "<span style=\"color: #7fbf7f; text-decoration-color: #7fbf7f\">│   │   │   │   </span><span style=\"font-weight: bold\">}</span>\n",
       "<span style=\"color: #7fbf7f; text-decoration-color: #7fbf7f\">│   │   │   </span><span style=\"font-weight: bold\">}</span>\n",
       "<span style=\"color: #7fbf7f; text-decoration-color: #7fbf7f\">│   │   </span><span style=\"font-weight: bold\">}</span>\n",
       "<span style=\"color: #7fbf7f; text-decoration-color: #7fbf7f\">│   </span><span style=\"font-weight: bold\">}</span>\n",
       "<span style=\"font-weight: bold\">}</span>\n",
       "</pre>\n"
      ],
      "text/plain": [
       "\u001b[1m{\u001b[0m\n",
       "\u001b[2;32m│   \u001b[0m\u001b[32m'open_book_qa'\u001b[0m: \u001b[1m{\u001b[0m\n",
       "\u001b[2;32m│   │   \u001b[0m\u001b[32m'test'\u001b[0m: \u001b[1m{\u001b[0m\n",
       "\u001b[2;32m│   │   │   \u001b[0m\u001b[32m'accuracy'\u001b[0m: \u001b[1m{\u001b[0m\n",
       "\u001b[2;32m│   │   │   │   \u001b[0m\u001b[32m'text-davinci-003'\u001b[0m: \u001b[1m{\u001b[0m\n",
       "\u001b[2;32m│   │   │   │   │   \u001b[0m\u001b[32m'None_kojima-01_The question was created by crowdsourcing with the goal to test AI systems. Multiple answers may be valid but only one answer will be regarded as correct. If you give more than one answer, it will be evaluated as wrong. Pick the one answer that is most likely to be correct for this question. Therefore, among A through D, the answer is'\u001b[0m: \u001b[1;36m0.047619\u001b[0m,\n",
       "\u001b[2;32m│   │   │   │   │   \u001b[0m\u001b[32m'None_kojima-01_kojima-A-D'\u001b[0m: \u001b[1;36m0.0\u001b[0m\n",
       "\u001b[2;32m│   │   │   │   \u001b[0m\u001b[1m}\u001b[0m\n",
       "\u001b[2;32m│   │   │   \u001b[0m\u001b[1m}\u001b[0m\n",
       "\u001b[2;32m│   │   \u001b[0m\u001b[1m}\u001b[0m\n",
       "\u001b[2;32m│   \u001b[0m\u001b[1m}\u001b[0m\n",
       "\u001b[1m}\u001b[0m\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "test = Collection.to_Collection(extract,\"open_book_qa\",'test','file_test')\n",
    "eval = test.evaluate()\n",
    "from rich.pretty import pprint\n",
    "pprint(eval)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [],
   "source": [
    "\"\"\"Da Vinci\"\"\"\n",
    "\n",
    "llm = OpenAI(temperature=.0,model_name=\"text-davinci-003\")  \n",
    "\n",
    "reflect_template = \"\"\"\n",
    "    Question: {question}\n",
    "    Answer_choices: {answer_choices}\n",
    "\n",
    "    {cot_trigger} {cot}\n",
    "    {answer_extraction} {answer}\n",
    "    \n",
    "    Reflection: {reflection_prompt}\n",
    "    \"\"\"\n",
    "reflect_prompt_template = PromptTemplate(input_variables=[\"question\",\"answer_choices\",\"cot_trigger\",\"cot\",\"answer_extraction\",'answer','reflection_prompt'], template=reflect_template)\n",
    "reflect_chain = LLMChain(llm=llm, prompt=reflect_prompt_template,output_key=\"reflection\")\n",
    "\n",
    "#{instruction}\n",
    "extraction_template = \"\"\"\n",
    "\n",
    "    Question: {question}\n",
    "    Answer_choices: {answer_choices}\n",
    "\n",
    "    {cot_trigger} {cot}\n",
    "    {answer_extraction} {answer}\n",
    "    \n",
    "    Reflection: {reflection_prompt}\n",
    "    {reflection}\n",
    "\n",
    "    {reflect_answer_extraction}\n",
    "    \"\"\"\n",
    "    #Get reflection\n",
    "ans_prompt_template = PromptTemplate(input_variables=[\"question\",\"answer_choices\",\"cot_trigger\",\"cot\",\"answer_extraction\",'answer','reflection_prompt','reflection','reflect_answer_extraction'], template=extraction_template)\n",
    "reflect_answer_chain = LLMChain(llm=llm, prompt=ans_prompt_template,output_key=\"reflection_answer\")\n",
    "\n",
    "    # This is the overall chain where we run these two chains in sequence.\n",
    "from langchain.chains import SequentialChain\n",
    "reflect_overall_chain = SequentialChain(chains=[reflect_chain, reflect_answer_chain],input_variables=[\"question\",\"answer_choices\",\"cot_trigger\",\"answer_extraction\",'cot','answer','reflection_prompt','reflect_answer_extraction'],\n",
    "        output_variables=[\"reflection\", \"reflection_answer\"],\n",
    "        verbose=True)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [],
   "source": [
    "#llm = ChatOpenAI(temperature=.0,model_name=\"gpt-4\")   #text-davinci-003\n",
    "llm = ChatOpenAI(temperature=.0,model_name=\"gpt-3.5-turbo\")  \n",
    "\n",
    "reflect_template = \"\"\"\n",
    "    Question: {question}\n",
    "    Answer_choices: {answer_choices}\n",
    "\n",
    "    {cot_trigger} {cot}\n",
    "    {answer_extraction} {answer}\n",
    "    \n",
    "    Reflection: {reflection_prompt}\n",
    "    \"\"\"\n",
    "reflect_prompt_template = PromptTemplate(input_variables=[\"question\",\"answer_choices\",\"cot_trigger\",\"cot\",\"answer_extraction\",'answer','reflection_prompt'], template=reflect_template)\n",
    "reflect_chain = LLMChain(llm=llm, prompt=reflect_prompt_template,output_key=\"reflection\")\n",
    "\n",
    "#{instruction}\n",
    "extraction_template = \"\"\"\n",
    "\n",
    "    Question: {question}\n",
    "    Answer_choices: {answer_choices}\n",
    "\n",
    "    {cot_trigger} {cot}\n",
    "    {answer_extraction} {answer}\n",
    "    \n",
    "    Reflection: {reflection_prompt}\n",
    "    {reflection}\n",
    "\n",
    "    {reflect_answer_extraction}\n",
    "    \"\"\"\n",
    "    #Get reflection\n",
    "ans_prompt_template = PromptTemplate(input_variables=[\"question\",\"answer_choices\",\"cot_trigger\",\"cot\",\"answer_extraction\",'answer','reflection_prompt','reflection','reflect_answer_extraction'], template=extraction_template)\n",
    "reflect_answer_chain = LLMChain(llm=llm, prompt=ans_prompt_template,output_key=\"reflection_answer\")\n",
    "\n",
    "    # This is the overall chain where we run these two chains in sequence.\n",
    "from langchain.chains import SequentialChain\n",
    "reflect_overall_chain = SequentialChain(chains=[reflect_chain, reflect_answer_chain],input_variables=[\"question\",\"answer_choices\",\"cot_trigger\",\"answer_extraction\",'cot','answer','reflection_prompt','reflect_answer_extraction'],\n",
    "        output_variables=[\"reflection\", \"reflection_answer\"],\n",
    "        verbose=True)\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [],
   "source": [
    "instruction = \"Answer the following question through step-by-step reasoning.\"\n",
    "question = \"Animals may fight, make threatening sounds, and act aggressively toward members of the same species. These behaviors usually occur as the result of\",\n",
    "answer_choices = [\n",
    "                    \"competition\",\n",
    "                    \"conservation\",\n",
    "                    \"decomposition\",\n",
    "                    \"pollution\"\n",
    "                ]\n",
    "cot_trigger = \"Answer: Let's think step by step.\"\n",
    "cot = \"Aggression is needed to defend something, one needs to defend their spot when they are in competition\"\n",
    "answer_extraction = \"Therefore, the answer is\"\n",
    "answer = \"competition\"\n",
    "reflection_prompt = \"do you agree with the cot yes or no\"\n",
    "reflection = \"Great reasoning mate!\"\n",
    "reflect_answer_extraction = \"Based on the text above the answer is:\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "    Question: ('Animals may fight, make threatening sounds, and act aggressively toward members of the same species. These behaviors usually occur as the result of',)\n",
      "    Answer_choices: ['competition', 'conservation', 'decomposition', 'pollution']\n",
      "\n",
      "    Answer: Let's think step by step.Aggression is needed to defend something, one needs to defend their spot when they are in competition\n",
      "    Therefore, the answer is competition\n",
      "    \n",
      "    Reflection: do you agree with the cot yes or no\n",
      "    \n"
     ]
    }
   ],
   "source": [
    "print(reflect_prompt_template.format(question=question,answer_choices=answer_choices,cot_trigger=cot_trigger,\n",
    "                                     cot=cot,answer_extraction=answer_extraction,\n",
    "                                     answer=answer,reflection_prompt=reflection_prompt))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "    Question: ('Animals may fight, make threatening sounds, and act aggressively toward members of the same species. These behaviors usually occur as the result of',)\n",
      "    Answer_choices: ['competition', 'conservation', 'decomposition', 'pollution']\n",
      "\n",
      "    Answer: Let's think step by step. Aggression is needed to defend something, one needs to defend their spot when they are in competition\n",
      "    Therefore, the answer is competition\n",
      "    \n",
      "    Reflection: do you agree with the cot yes or no\n",
      "    Great reasoning mate!\n",
      "\n",
      "    Based on the text above the answer is:\n",
      "    \n"
     ]
    }
   ],
   "source": [
    "print(ans_prompt_template.format(question=question,answer_choices=answer_choices,cot_trigger=cot_trigger,\n",
    "                                     cot=cot,answer_extraction=answer_extraction,\n",
    "                                     answer=answer,reflection_prompt=reflection_prompt,\n",
    "                                     reflection=reflection,reflect_answer_extraction=reflect_answer_extraction))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "coll = Collection.load_thoughtsource_100(names='strategy_qa',load_pregenerated_cots=True) #random_sample=False?\n",
    "coll = coll.select(split=\"all\", number_samples=1)\n",
    "coll.select_generated_cots(cot_trigger = \"kojima-01\", api_service='cohere') #have one"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "| Name        |   Train | Valid   | Test   |\n",
       "|-------------|---------|---------|--------|\n",
       "| strategy_qa |       1 | -       | -      |\n",
       "\n",
       "Not loaded: ['aqua', 'asdiv', 'commonsense_qa', 'entailment_bank', 'gsm8k', 'mawps', 'med_qa', 'medmc_qa', 'open_book_qa', 'pubmed_qa', 'qed', 'svamp', 'worldtree']"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "coll"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Run reflection"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "input_dict = {\n",
    "    \"cot_trigger\": \"Answer: Let's think step by step.\",\n",
    "    \"answer_extraction\": \"Therefore, among A through D, the answer is\", \n",
    "    'answer':\"\", \n",
    "    'cot': \"\", \n",
    "    'reflection_prompt':\"Do you have any reason to believe that the reasoning or the answer might be wrong? Answer with one word Yes or No\",\n",
    "    'reflect_answer_extraction':'Based on the reflection, what is the definite answer?'\n",
    "}\n",
    "metareason = coll.metareason_flexible(chain=reflect_overall_chain,input_dict=input_dict)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#These prompts only let the model say no reason to believe the answer is wrong"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "input_dict = {\n",
    "    \"cot_trigger\": \"Answer: Let's think step by step.\",\n",
    "    \"answer_extraction\": \"Therefore, among A through D, the answer is\", \n",
    "    'answer':\"\", \n",
    "    'cot': \"\", \n",
    "    'reflection_prompt':\"The goal is to correct the Answer if needed, let's think step by step\",\n",
    "    'reflect_answer_extraction':'Based on the reflection, what is the definite answer?',\n",
    "    'model_name':\"gpt-3.5-turbo\"\n",
    "}\n",
    "metareason = coll.metareason_flexible(chain=reflect_overall_chain,input_dict=input_dict)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "9f1983f0119042709d99382b9715b866",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/20 [00:00<?, ?ex/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"font-weight: bold\">{</span>\n",
       "<span style=\"color: #7fbf7f; text-decoration-color: #7fbf7f\">│   </span><span style=\"color: #008000; text-decoration-color: #008000\">'strategy_qa'</span>: <span style=\"font-weight: bold\">{</span>\n",
       "<span style=\"color: #7fbf7f; text-decoration-color: #7fbf7f\">│   │   </span><span style=\"color: #008000; text-decoration-color: #008000\">'train'</span>: <span style=\"font-weight: bold\">{</span>\n",
       "<span style=\"color: #7fbf7f; text-decoration-color: #7fbf7f\">│   │   │   </span><span style=\"color: #008000; text-decoration-color: #008000\">'accuracy'</span>: <span style=\"font-weight: bold\">{</span>\n",
       "<span style=\"color: #7fbf7f; text-decoration-color: #7fbf7f\">│   │   │   │   </span><span style=\"color: #008000; text-decoration-color: #008000\">''</span>: <span style=\"font-weight: bold\">{</span><span style=\"color: #008000; text-decoration-color: #008000\">\"_Answer: Let's think step by step._Therefore, among A through D, the answer is\"</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0.45</span><span style=\"font-weight: bold\">}</span>,\n",
       "<span style=\"color: #7fbf7f; text-decoration-color: #7fbf7f\">│   │   │   │   </span><span style=\"color: #008000; text-decoration-color: #008000\">'command-xlarge-nightly'</span>: <span style=\"font-weight: bold\">{</span><span style=\"color: #008000; text-decoration-color: #008000\">'None_kojima-01_kojima-yes-no'</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0.6</span><span style=\"font-weight: bold\">}</span>\n",
       "<span style=\"color: #7fbf7f; text-decoration-color: #7fbf7f\">│   │   │   </span><span style=\"font-weight: bold\">}</span>\n",
       "<span style=\"color: #7fbf7f; text-decoration-color: #7fbf7f\">│   │   </span><span style=\"font-weight: bold\">}</span>\n",
       "<span style=\"color: #7fbf7f; text-decoration-color: #7fbf7f\">│   </span><span style=\"font-weight: bold\">}</span>\n",
       "<span style=\"font-weight: bold\">}</span>\n",
       "</pre>\n"
      ],
      "text/plain": [
       "\u001b[1m{\u001b[0m\n",
       "\u001b[2;32m│   \u001b[0m\u001b[32m'strategy_qa'\u001b[0m: \u001b[1m{\u001b[0m\n",
       "\u001b[2;32m│   │   \u001b[0m\u001b[32m'train'\u001b[0m: \u001b[1m{\u001b[0m\n",
       "\u001b[2;32m│   │   │   \u001b[0m\u001b[32m'accuracy'\u001b[0m: \u001b[1m{\u001b[0m\n",
       "\u001b[2;32m│   │   │   │   \u001b[0m\u001b[32m''\u001b[0m: \u001b[1m{\u001b[0m\u001b[32m\"_Answer: Let's think step by step._Therefore, among A through D, the answer is\"\u001b[0m: \u001b[1;36m0.45\u001b[0m\u001b[1m}\u001b[0m,\n",
       "\u001b[2;32m│   │   │   │   \u001b[0m\u001b[32m'command-xlarge-nightly'\u001b[0m: \u001b[1m{\u001b[0m\u001b[32m'None_kojima-01_kojima-yes-no'\u001b[0m: \u001b[1;36m0.6\u001b[0m\u001b[1m}\u001b[0m\n",
       "\u001b[2;32m│   │   │   \u001b[0m\u001b[1m}\u001b[0m\n",
       "\u001b[2;32m│   │   \u001b[0m\u001b[1m}\u001b[0m\n",
       "\u001b[2;32m│   \u001b[0m\u001b[1m}\u001b[0m\n",
       "\u001b[1m}\u001b[0m\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "#sloppy metareasoning leads to a performance decrease"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\"\"\"Use TS_hard dataset\"\"\"\n",
    "ts_hard = Collection.from_json('ts_hard_v1.json')\n",
    "ts_sqa = ts_hard.select(split=\"all\", number_samples=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\"\"\"2/11 correct\"\"\"\n",
    "\n",
    "input_dict = {\n",
    "    \"cot_trigger\": \"Answer: Let's think step by step.\",\n",
    "    \"answer_extraction\": \"Therefore, the answer is\", \n",
    "    'answer':\"\", \n",
    "    'cot': \"\", \n",
    "    'reflection_prompt':\"The question may try to trick the respondent, revise the previous answer\",\n",
    "    'reflect_answer_extraction':'What is the final answer (true/false)?',\n",
    "    'model_name':\"gpt-3.5-turbo\"\n",
    "}\n",
    "metareason = ts_hard.metareason_flexible(chain=reflect_overall_chain,input_dict=input_dict)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\"\"\".36 corrected but a bit misleading\"\"\" \n",
    "input_dict = {\n",
    "    \"cot_trigger\": \"Answer: Let's think step by step.\",\n",
    "    \"answer_extraction\": \"Therefore, the answer is\", \n",
    "    'answer':\"\", \n",
    "    'cot': \"\", \n",
    "    'reflection_prompt':\"The question may try to trick the respondent and is designed to have one best answer. With this information, revise the previous answer\",\n",
    "    'reflect_answer_extraction':'What is the final answer? Answer with one word (true/false)?',\n",
    "    'model_name':\"gpt-3.5-turbo\"\n",
    "}\n",
    "metareason = ts_hard.metareason_flexible(chain=reflect_overall_chain,input_dict=input_dict)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\"\"\"18% accuracy\"\"\" \n",
    "input_dict = {\n",
    "    \"cot_trigger\": \"Answer: Let's think step by step.\",\n",
    "    \"answer_extraction\": \"Therefore, the answer is\", \n",
    "    'answer':\"\", \n",
    "    'cot': \"\", \n",
    "    'reflection_prompt':\"The question may try to trick the respondent and is designed to have one best answer. With this information, let's revise the previous answer step-by-step\",\n",
    "    'reflect_answer_extraction':'What is the final answer? Answer with one word (true/false)?',\n",
    "    'model_name':\"gpt-3.5-turbo\"\n",
    "}\n",
    "metareason = ts_hard.metareason_flexible(chain=reflect_overall_chain,input_dict=input_dict)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\"\"\"DaVinci 0.09 accuracy\"\"\" \n",
    "input_dict = {\n",
    "    \"cot_trigger\": \"Answer: Let's think step by step.\",\n",
    "    \"answer_extraction\": \"Therefore, the answer is\", \n",
    "    'answer':\"\", \n",
    "    'cot': \"\", \n",
    "    'reflection_prompt':\"The question may try to trick the respondent and is designed to have one best answer. With this information, revise the previous answer\",\n",
    "    'reflect_answer_extraction':'What is the final answer? Answer with one word (true/false)?',\n",
    "    'model_name':\"text-davinci-003\"\n",
    "}\n",
    "metareason = ts_hard.metareason_flexible(chain=reflect_overall_chain,input_dict=input_dict)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\"\"\"ChatGPT as language model I have no access to the creator's answer\"\"\"\n",
    "\n",
    "input_dict = {\n",
    "    \"cot_trigger\": \"Answer: Let's think step by step.\",\n",
    "    \"answer_extraction\": \"Therefore, the answer is\", \n",
    "    'answer':\"\", \n",
    "    'cot': \"\", \n",
    "    'reflection_prompt':\"The question was created by crowdsourcing with the goal to test AI systems. Multiple answers may be multiple valid but only one will be regarded as correct. Pick the one answer that the creator of the question picked. If you give more than one, you are evaluated as wrong.\",\n",
    "    'reflect_answer_extraction':'What is the final answer? Answer with one word (true/false)?',\n",
    "    'model_name':\"gpt-3.5-turbo\"\n",
    "}\n",
    "metareason = ts_hard.metareason_flexible(chain=reflect_overall_chain,input_dict=input_dict)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\"\"\"27% accuracy\"\"\"\n",
    "input_dict = {\n",
    "    \"cot_trigger\": \"Answer: Let's think step by step.\",\n",
    "    \"answer_extraction\": \"Therefore, the answer is\", \n",
    "    'answer':\"\", \n",
    "    'cot': \"\", \n",
    "    'reflection_prompt':\"The question was created by crowdsourcing with the goal to test AI systems. Multiple answers may be valid but only one answer will be regarded as correct. If you give more than one answer, it will be evaluated as wrong. Pick the one answer that is most likely for this question. \",\n",
    "    'reflect_answer_extraction':'What is the final answer? Answer with one word (true/false)?',\n",
    "    'model_name':\"gpt-3.5-turbo\"\n",
    "}\n",
    "metareason = ts_hard.metareason_flexible(chain=reflect_overall_chain,input_dict=input_dict)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [],
   "source": [
    "test = Collection.to_Collection(metareason,\"strategy_qa\",'train','file_test')\n",
    "eval = test.evaluate()\n",
    "from rich.pretty import pprint\n",
    "pprint(eval)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "6e864a1cef684a7cb831eec90881a3b9",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/11 [00:00<?, ?ex/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "| Name        |   Train | Valid   | Test   |\n",
       "|-------------|---------|---------|--------|\n",
       "| strategy_qa |      11 | -       | -      |\n",
       "\n",
       "Not loaded: ['aqua', 'asdiv', 'commonsense_qa', 'entailment_bank', 'gsm8k', 'mawps', 'med_qa', 'medmc_qa', 'open_book_qa', 'pubmed_qa', 'qed', 'svamp', 'worldtree']"
      ]
     },
     "execution_count": 55,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test.evaluate()\n",
    "test.dump('file_test.json')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "| Name         | Train   | Valid   |   Test |\n",
       "|--------------|---------|---------|--------|\n",
       "| open_book_qa | -       | -       |     21 |\n",
       "\n",
       "Not loaded: ['aqua', 'asdiv', 'commonsense_qa', 'entailment_bank', 'gsm8k', 'mawps', 'med_qa', 'medmc_qa', 'pubmed_qa', 'qed', 'strategy_qa', 'svamp', 'worldtree']"
      ]
     },
     "execution_count": 57,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ts_hard = Collection.from_json('ts_hard_v1.json')\n",
    "ts_sqa = ts_hard.select(split=\"all\", number_samples=1)\n",
    "ts_hard.unload_datasets([\"med_qa\",\"medmc_qa\",\"commonsense_qa\",\"strategy_qa\",\"worldtree\"])\n",
    "ts_hard"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\"\"\"As an AI language model, I agree with the given answer\"\"\"\n",
    "input_dict = {\n",
    "    \"cot_trigger\": \"Answer: Let's think step by step.\",\n",
    "    \"answer_extraction\": \"Therefore, the answer is\", \n",
    "    'answer':\"\", \n",
    "    'cot': \"\", \n",
    "    'reflection_prompt':\"The question was created by crowdsourcing with the goal to test AI systems. Multiple answers may be valid but only one answer will be regarded as correct. If you give more than one answer, it will be evaluated as wrong. Pick the one answer that is most likely for this question.\",\n",
    "    'reflect_answer_extraction':'What is the final answer from A through D?',\n",
    "    'model_name':\"gpt-3.5-turbo\"\n",
    "}\n",
    "metareason = ts_hard.metareason_flexible(chain=reflect_overall_chain,input_dict=input_dict)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\"\"\"As an AI language model, I stand corrected., 14% accuracy\"\"\"\n",
    "input_dict = {\n",
    "    \"cot_trigger\": \"Answer: Let's think step by step.\",\n",
    "    \"answer_extraction\": \"Therefore, the answer is\", \n",
    "    'answer':\"\", \n",
    "    'cot': \"\", \n",
    "    'reflection_prompt':\"The Answer was given by a language model and may be incorrect. Multiple answers may be valid but only one answer is the best correct answer. If you give more than one answer, it will be evaluated as wrong. Revise, then pick the one answer that is most likely for this question.\",\n",
    "    'reflect_answer_extraction':'What is the final answer from A through D?',\n",
    "    'model_name':\"gpt-3.5-turbo\"\n",
    "}\n",
    "metareason = ts_hard.metareason_flexible(chain=reflect_overall_chain,input_dict=input_dict)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "ts_hard = Collection.from_json('ts_hard_v1.json')\n",
    "ts_hard.unload_datasets([\"med_qa\",\"medmc_qa\",\"commonsense_qa\",\"open_book_qa\",\"worldtree\"])\n",
    "\n",
    "\"\"\"9%\"\"\"\n",
    "input_dict = {\n",
    "    \"cot_trigger\": \"Answer: Let's think step by step.\",\n",
    "    \"answer_extraction\": \"Therefore, the answer is\", \n",
    "    'answer':\"\", \n",
    "    'cot': \"\", \n",
    "    'reflection_prompt':\"The Answer was given by a language model and may be incorrect. Multiple answers may be valid but only one answer is the best correct answer. If you give more than one answer, it will be evaluated as wrong. Revise, then pick the one answer that is most likely for this question.\",\n",
    "    'reflect_answer_extraction':'What is the final answer (true/false)?',\n",
    "    'model_name':\"gpt-3.5-turbo\"\n",
    "}\n",
    "metareason = ts_hard.metareason_flexible(chain=reflect_overall_chain,input_dict=input_dict)\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "efb6cf521f69460e8354664e5ff670c1",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/11 [00:00<?, ?ex/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"font-weight: bold\">{</span>\n",
       "<span style=\"color: #7fbf7f; text-decoration-color: #7fbf7f\">│   </span><span style=\"color: #008000; text-decoration-color: #008000\">'strategy_qa'</span>: <span style=\"font-weight: bold\">{</span>\n",
       "<span style=\"color: #7fbf7f; text-decoration-color: #7fbf7f\">│   │   </span><span style=\"color: #008000; text-decoration-color: #008000\">'train'</span>: <span style=\"font-weight: bold\">{</span>\n",
       "<span style=\"color: #7fbf7f; text-decoration-color: #7fbf7f\">│   │   │   </span><span style=\"color: #008000; text-decoration-color: #008000\">'accuracy'</span>: <span style=\"font-weight: bold\">{</span>\n",
       "<span style=\"color: #7fbf7f; text-decoration-color: #7fbf7f\">│   │   │   │   </span><span style=\"color: #008000; text-decoration-color: #008000\">''</span>: <span style=\"font-weight: bold\">{</span><span style=\"color: #008000; text-decoration-color: #008000\">\"_Answer: Let's think step by step._Therefore, the answer is\"</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0.090909</span><span style=\"font-weight: bold\">}</span>,\n",
       "<span style=\"color: #7fbf7f; text-decoration-color: #7fbf7f\">│   │   │   │   </span><span style=\"color: #008000; text-decoration-color: #008000\">'text-davinci-003'</span>: <span style=\"font-weight: bold\">{</span><span style=\"color: #008000; text-decoration-color: #008000\">'None_kojima-01_kojima-yes-no'</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0.0</span><span style=\"font-weight: bold\">}</span>\n",
       "<span style=\"color: #7fbf7f; text-decoration-color: #7fbf7f\">│   │   │   </span><span style=\"font-weight: bold\">}</span>\n",
       "<span style=\"color: #7fbf7f; text-decoration-color: #7fbf7f\">│   │   </span><span style=\"font-weight: bold\">}</span>\n",
       "<span style=\"color: #7fbf7f; text-decoration-color: #7fbf7f\">│   </span><span style=\"font-weight: bold\">}</span>\n",
       "<span style=\"font-weight: bold\">}</span>\n",
       "</pre>\n"
      ],
      "text/plain": [
       "\u001b[1m{\u001b[0m\n",
       "\u001b[2;32m│   \u001b[0m\u001b[32m'strategy_qa'\u001b[0m: \u001b[1m{\u001b[0m\n",
       "\u001b[2;32m│   │   \u001b[0m\u001b[32m'train'\u001b[0m: \u001b[1m{\u001b[0m\n",
       "\u001b[2;32m│   │   │   \u001b[0m\u001b[32m'accuracy'\u001b[0m: \u001b[1m{\u001b[0m\n",
       "\u001b[2;32m│   │   │   │   \u001b[0m\u001b[32m''\u001b[0m: \u001b[1m{\u001b[0m\u001b[32m\"_Answer: Let's think step by step._Therefore, the answer is\"\u001b[0m: \u001b[1;36m0.090909\u001b[0m\u001b[1m}\u001b[0m,\n",
       "\u001b[2;32m│   │   │   │   \u001b[0m\u001b[32m'text-davinci-003'\u001b[0m: \u001b[1m{\u001b[0m\u001b[32m'None_kojima-01_kojima-yes-no'\u001b[0m: \u001b[1;36m0.0\u001b[0m\u001b[1m}\u001b[0m\n",
       "\u001b[2;32m│   │   │   \u001b[0m\u001b[1m}\u001b[0m\n",
       "\u001b[2;32m│   │   \u001b[0m\u001b[1m}\u001b[0m\n",
       "\u001b[2;32m│   \u001b[0m\u001b[1m}\u001b[0m\n",
       "\u001b[1m}\u001b[0m\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "test = Collection.to_Collection(metareason,\"strategy_qa\",'train','file_test')\n",
    "eval = test.evaluate()\n",
    "from rich.pretty import pprint\n",
    "pprint(eval)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "4cd3ac7965264e6f8cb0dce2bc1316aa",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/11 [00:00<?, ?ex/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "ce834327ecad4e44986eee1a429c9cd7",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Creating json from Arrow format:   0%|          | 0/1 [00:00<?, ?ba/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "test.evaluate()\n",
    "test.dump('file_test.json')"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Overall review"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "from cot import Collection\n",
    "ts_hard = Collection.from_json('ts_hard_v1.json')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\"The question was created by crowdsourcing with the goal to test AI systems. Multiple answers may be valid but only one answer will be regarded as correct. If you give more than one answer, it will be evaluated as wrong. Pick the one answer that is most likely for this question. \""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\"The goal is to correct the Answer (generated by a language model) if needed, let's think step by step\" \n",
    "\"The question may try to trick the respondent and is designed to have one best answer. With this information, revise the previous answer made by a language model\"\n",
    "\"The Answer was given by a language model and may be incorrect. Multiple answers may be valid but only one answer is the best correct answer. If you give more than one answer, it will be evaluated as wrong. Revise, then pick the one answer that is most likely for this question.\""
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## First prompt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#check answer extraction\n",
    "input_dict = {\n",
    "    \"cot_trigger\": \"Answer: Let's think step by step.\",\n",
    "    \"answer_extraction\": \"Therefore, the answer is\", \n",
    "    'answer':\"\", \n",
    "    'cot': \"\", \n",
    "    'reflection_prompt':\"The goal is to correct the Answer (generated by a language model) if needed, let's think step by step\" ,\n",
    "    'reflect_answer_extraction':'Based on the reflection, the answer is therefore',\n",
    "    'model_name':\"gpt-3.5-turbo\"\n",
    "}\n",
    "ts_hard.metareason_flexible(chain=reflect_overall_chain,input_dict=input_dict)\n",
    "ts_hard.evaluate()\n",
    "ts_hard.dump(\"first_prompt.json\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "ts_hard.evaluate()\n",
    "ts_hard.dump(\"first_prompt_evaluated.json\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "dae2304c347d4c7eb6f1d98cec3d8c78",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/12 [00:00<?, ?ex/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "34430a7985c746b1838d7a1d381f8627",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/7 [00:00<?, ?ex/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "32b62e694cfb406a8c2a0779f8fb2ee6",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/11 [00:00<?, ?ex/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "66bbed75f3ec4b29b3452a1703193620",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/21 [00:00<?, ?ex/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "ca4c4b32a4dd4652a08862409003bf3a",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/20 [00:00<?, ?ex/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "{'commonsense_qa': {'validation': {'accuracy': {'': {\"_The goal is to correct the Answer (generated by a language model) if needed, let's think step by step_Based on the reflection, the answer is therefore\": 0.0},\n",
       "    'text-davinci-003': {'None_kojima-01_kojima-A-E': 0.0}}}},\n",
       " 'worldtree': {'test': {'accuracy': {'': {\"_The goal is to correct the Answer (generated by a language model) if needed, let's think step by step_Based on the reflection, the answer is therefore\": 0.142857},\n",
       "    'text-davinci-003': {'None_kojima-01_kojima-A-D': 0.0}}}},\n",
       " 'strategy_qa': {'train': {'accuracy': {'': {\"_The goal is to correct the Answer (generated by a language model) if needed, let's think step by step_Based on the reflection, the answer is therefore\": 0.0},\n",
       "    'text-davinci-003': {'None_kojima-01_kojima-yes-no': 0.0}}}},\n",
       " 'open_book_qa': {'test': {'accuracy': {'': {\"_The goal is to correct the Answer (generated by a language model) if needed, let's think step by step_Based on the reflection, the answer is therefore\": 0.142857},\n",
       "    'text-davinci-003': {'None_kojima-01_kojima-A-D': 0.0}}}},\n",
       " 'med_qa': {'test': {'accuracy': {'': {\"_The goal is to correct the Answer (generated by a language model) if needed, let's think step by step_Based on the reflection, the answer is therefore\": 0.05},\n",
       "    'text-davinci-003': {'None_kojima-01_kojima-A-E': 0.0}}}}}"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ts_hard.evaluate()"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Second prompt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/anaconda3/lib/python3.9/site-packages/scipy/__init__.py:146: UserWarning: A NumPy version >=1.16.5 and <1.23.0 is required for this version of SciPy (detected version 1.24.2\n",
      "  warnings.warn(f\"A NumPy version >={np_minversion} and <{np_maxversion}\"\n",
      "[nltk_data] Downloading package punkt to\n",
      "[nltk_data]     /Users/robertpraas/nltk_data...\n",
      "[nltk_data]   Package punkt is already up-to-date!\n"
     ]
    }
   ],
   "source": [
    "ts_hard = Collection.from_json('ts_hard_v1.json')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "input_dict = {\n",
    "    \"cot_trigger\": \"Answer: Let's think step by step.\",\n",
    "    \"answer_extraction\": \"Therefore, the answer is\", \n",
    "    'answer':\"\", \n",
    "    'cot': \"\", \n",
    "    'reflection_prompt':\"The question may try to trick the respondent and is designed to have one best answer. With this information, revise the previous answer made by a language model\" ,\n",
    "    'reflect_answer_extraction':'Based on the reflection, the answer is therefore',\n",
    "    'model_name':\"gpt-3.5-turbo\"\n",
    "}\n",
    "ts_hard.metareason_flexible(chain=reflect_overall_chain,input_dict=input_dict)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "0e1a4fc7fed24db385c04062896def82",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Creating json from Arrow format:   0%|          | 0/1 [00:00<?, ?ba/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "5d8dde3066944b19b7e1274839d6cce9",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Creating json from Arrow format:   0%|          | 0/1 [00:00<?, ?ba/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "36b767ec2e7f408898006400c7f8fad8",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Creating json from Arrow format:   0%|          | 0/1 [00:00<?, ?ba/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "debd5900d02748dc8ab603e3f4bf49d5",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Creating json from Arrow format:   0%|          | 0/1 [00:00<?, ?ba/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "23a142804fbe431794587b4cf31471c9",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Creating json from Arrow format:   0%|          | 0/1 [00:00<?, ?ba/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "ts_hard.dump(\"second_prompt.json\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "c6fb8d1a36134823ae92ba8a24b2190c",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/12 [00:00<?, ?ex/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "0b1f129895fb44bca8f53c65d182c619",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/7 [00:00<?, ?ex/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "398e5e4fd02943d3a863e9ec3edc7ef0",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/11 [00:00<?, ?ex/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "64e0ab3c7a7448a2bbe41c68cf815e1f",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/21 [00:00<?, ?ex/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "e28d2f0ad17845fbb75e796b1fb8e5c7",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/20 [00:00<?, ?ex/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "{'commonsense_qa': {'validation': {'accuracy': {'': {\"_Answer: Let's think step by step._Therefore, the answer is\": 0.083333},\n",
       "    'text-davinci-003': {'None_kojima-01_kojima-A-E': 0.0}}}},\n",
       " 'worldtree': {'test': {'accuracy': {'': {\"_Answer: Let's think step by step._Therefore, the answer is\": 0.142857},\n",
       "    'text-davinci-003': {'None_kojima-01_kojima-A-D': 0.0}}}},\n",
       " 'strategy_qa': {'train': {'accuracy': {'': {\"_Answer: Let's think step by step._Therefore, the answer is\": 0.090909},\n",
       "    'text-davinci-003': {'None_kojima-01_kojima-yes-no': 0.0}}}},\n",
       " 'open_book_qa': {'test': {'accuracy': {'': {\"_Answer: Let's think step by step._Therefore, the answer is\": 0.047619},\n",
       "    'text-davinci-003': {'None_kojima-01_kojima-A-D': 0.0}}}},\n",
       " 'med_qa': {'test': {'accuracy': {'': {\"_Answer: Let's think step by step._Therefore, the answer is\": 0.0},\n",
       "    'text-davinci-003': {'None_kojima-01_kojima-A-E': 0.0}}}}}"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ts_hard.evaluate()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "cbf3b3bd51294758938bc67fd6fc7f99",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Creating json from Arrow format:   0%|          | 0/1 [00:00<?, ?ba/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "7bb2f9a2523448cb8468db73d9d3f0cf",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Creating json from Arrow format:   0%|          | 0/1 [00:00<?, ?ba/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "af66c6a2667543869e80268d77dde330",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Creating json from Arrow format:   0%|          | 0/1 [00:00<?, ?ba/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "d7d5f4bca4b946f8a8f6eb7026146ca6",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Creating json from Arrow format:   0%|          | 0/1 [00:00<?, ?ba/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "0fd0ec3a766c4367bec7197411ee1bab",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Creating json from Arrow format:   0%|          | 0/1 [00:00<?, ?ba/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "ts_hard.dump(\"second_evaluated\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "ts_hard_2 = Collection.from_json('ts_hard_v1.json')\n",
    "input_dict = {\n",
    "    \"cot_trigger\": \"Answer: Let's think step by step.\",\n",
    "    \"answer_extraction\": \"Therefore, the answer is\", \n",
    "    'answer':\"\", \n",
    "    'cot': \"\", \n",
    "    'reflection_prompt':\"The Answer was given by a language model and may be incorrect. Multiple answers may be valid but only one answer is the best correct answer. If you give more than one answer, it will be evaluated as wrong. Revise, then pick the one answer that is most likely for this question.\",\n",
    "    'reflect_answer_extraction':'Based on the reflection, the answer is therefore',\n",
    "    'model_name':\"gpt-3.5-turbo\"\n",
    "}\n",
    "ts_hard_2.metareason_flexible(chain=reflect_overall_chain,input_dict=input_dict)\n",
    "ts_hard_2.dump(\"third_prompt.json\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "6f3dbc3673854e598337587aa408a24f",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/12 [00:00<?, ?ex/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "c7d2b84d65ed44d3829fd0418688801f",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/7 [00:00<?, ?ex/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "4e589d98b2ca4d299ca806670bb6c52a",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/11 [00:00<?, ?ex/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "00d0c5fac62d41f69478fe0763858a5d",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/21 [00:00<?, ?ex/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "fc2811ac1c254a209846b2a98a3e5a81",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/20 [00:00<?, ?ex/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "{'commonsense_qa': {'validation': {'accuracy': {'': {\"_Answer: Let's think step by step._Therefore, the answer is\": 0.333333},\n",
       "    'text-davinci-003': {'None_kojima-01_kojima-A-E': 0.0}}}},\n",
       " 'worldtree': {'test': {'accuracy': {'': {\"_Answer: Let's think step by step._Therefore, the answer is\": 0.428571},\n",
       "    'text-davinci-003': {'None_kojima-01_kojima-A-D': 0.0}}}},\n",
       " 'strategy_qa': {'train': {'accuracy': {'': {\"_Answer: Let's think step by step._Therefore, the answer is\": 0.0},\n",
       "    'text-davinci-003': {'None_kojima-01_kojima-yes-no': 0.0}}}},\n",
       " 'open_book_qa': {'test': {'accuracy': {'': {\"_Answer: Let's think step by step._Therefore, the answer is\": 0.238095},\n",
       "    'text-davinci-003': {'None_kojima-01_kojima-A-D': 0.0}}}},\n",
       " 'med_qa': {'test': {'accuracy': {'': {\"_Answer: Let's think step by step._Therefore, the answer is\": 0.05},\n",
       "    'text-davinci-003': {'None_kojima-01_kojima-A-E': 0.0}}}}}"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ts_hard_2.evaluate()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "3fbfe626ef014ff1bcb9086db9418945",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Creating json from Arrow format:   0%|          | 0/1 [00:00<?, ?ba/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "dec660213c3545879b0f90c9881455dd",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Creating json from Arrow format:   0%|          | 0/1 [00:00<?, ?ba/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "3906e48d706f4b63bbedd120726ff6b7",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Creating json from Arrow format:   0%|          | 0/1 [00:00<?, ?ba/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "4c0b51d7a43845c3ab1048065b8055f6",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Creating json from Arrow format:   0%|          | 0/1 [00:00<?, ?ba/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "948c206954e443eeab311f4efa79a63e",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Creating json from Arrow format:   0%|          | 0/1 [00:00<?, ?ba/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "ts_hard_2.dump(\"third_evaluated\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#next step: inspect if other ans extraction can boost"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/anaconda3/lib/python3.9/site-packages/scipy/__init__.py:146: UserWarning: A NumPy version >=1.16.5 and <1.23.0 is required for this version of SciPy (detected version 1.24.2\n",
      "  warnings.warn(f\"A NumPy version >={np_minversion} and <{np_maxversion}\"\n",
      "[nltk_data] Downloading package punkt to\n",
      "[nltk_data]     /Users/robertpraas/nltk_data...\n",
      "[nltk_data]   Package punkt is already up-to-date!\n"
     ]
    },
    {
     "ename": "IndexError",
     "evalue": "list index out of range",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mIndexError\u001b[0m                                Traceback (most recent call last)",
      "\u001b[1;32m/Users/robertpraas/Desktop/ThoughtSource/notebooks/summary_experiments.ipynb Cell 58\u001b[0m in \u001b[0;36m<cell line: 3>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      <a href='vscode-notebook-cell:/Users/robertpraas/Desktop/ThoughtSource/notebooks/summary_experiments.ipynb#Y113sZmlsZQ%3D%3D?line=0'>1</a>\u001b[0m \u001b[39mfrom\u001b[39;00m \u001b[39mcot\u001b[39;00m\u001b[39m.\u001b[39;00m\u001b[39mstats\u001b[39;00m \u001b[39mimport\u001b[39;00m evaluation_as_table\n\u001b[1;32m      <a href='vscode-notebook-cell:/Users/robertpraas/Desktop/ThoughtSource/notebooks/summary_experiments.ipynb#Y113sZmlsZQ%3D%3D?line=1'>2</a>\u001b[0m test \u001b[39m=\u001b[39m Collection\u001b[39m.\u001b[39mfrom_json(\u001b[39m\"\u001b[39m\u001b[39mthird_evaluated.json\u001b[39m\u001b[39m\"\u001b[39m)\n\u001b[0;32m----> <a href='vscode-notebook-cell:/Users/robertpraas/Desktop/ThoughtSource/notebooks/summary_experiments.ipynb#Y113sZmlsZQ%3D%3D?line=2'>3</a>\u001b[0m evaluation_as_table(test)\n",
      "File \u001b[0;32m~/Desktop/ThoughtSource/libs/cot/cot/stats.py:353\u001b[0m, in \u001b[0;36mevaluation_as_table\u001b[0;34m(eval)\u001b[0m\n\u001b[1;32m    351\u001b[0m \u001b[39mdef\u001b[39;00m \u001b[39mevaluation_as_table\u001b[39m(\u001b[39meval\u001b[39m:\u001b[39mdict\u001b[39m):\n\u001b[1;32m    352\u001b[0m     \u001b[39mimport\u001b[39;00m \u001b[39mpandas\u001b[39;00m \u001b[39mas\u001b[39;00m \u001b[39mpd\u001b[39;00m\n\u001b[0;32m--> 353\u001b[0m     eval_dict \u001b[39m=\u001b[39m pd\u001b[39m.\u001b[39;49mjson_normalize(\u001b[39meval\u001b[39;49m)\u001b[39m.\u001b[39;49mto_dict(\u001b[39m'\u001b[39;49m\u001b[39mrecords\u001b[39;49m\u001b[39m'\u001b[39;49m)[\u001b[39m0\u001b[39;49m]\n\u001b[1;32m    354\u001b[0m     eval_list \u001b[39m=\u001b[39m \u001b[39mlist\u001b[39m(eval_dict\u001b[39m.\u001b[39mkeys())\n\u001b[1;32m    355\u001b[0m     \u001b[39mprint\u001b[39m(eval_list)\n",
      "\u001b[0;31mIndexError\u001b[0m: list index out of range"
     ]
    }
   ],
   "source": [
    "from cot.stats import evaluation_as_table\n",
    "test = Collection.from_json(\"third_evaluated.json\")\n",
    "evaluation_as_table(test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# join strings from a list with underscore\n",
    "def join_strings(list_of_strings):\n",
    "    if list_of_strings is None:\n",
    "        list_of_strings = [\"None\"]\n",
    "    if isinstance(list_of_strings, str):\n",
    "        list_of_strings = [list_of_strings]\n",
    "    joined_string = \"\"\n",
    "    for string in list_of_strings:\n",
    "        joined_string += (\"-\" + str(string))\n",
    "    # delete first underscore\n",
    "    joined_string = joined_string[1:]\n",
    "    return(joined_string)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "coll.dump(\"thoughtsource_100\" + \"_\" + config['api_service'] + \"_\" + config['engine'].replace(\"/\", \"_\") + \"_\" + join_strings(config[\"instruction_keys\"]) + \"_\" + join_strings(config[\"cot_trigger_keys\"]) + \".json\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "ts_hard = Collection.from_json('ts_hard_v1.json')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "| Name           | Train   | Valid   | Test   |\n",
       "|----------------|---------|---------|--------|\n",
       "| commonsense_qa | -       | 12      | -      |\n",
       "| worldtree      | -       | -       | 7      |\n",
       "| strategy_qa    | 11      | -       | -      |\n",
       "| open_book_qa   | -       | -       | 21     |\n",
       "| med_qa         | -       | -       | 20     |\n",
       "\n",
       "Not loaded: ['aqua', 'asdiv', 'entailment_bank', 'gsm8k', 'mawps', 'medmc_qa', 'pubmed_qa', 'qed', 'svamp']"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ts_hard"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Prompt 3 and GPT-4"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "ts_hard = Collection.from_json('ts_hard_v1.json')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "input_dict = {\n",
    "    \"cot_trigger\": \"Answer: Let's think step by step.\",\n",
    "    \"answer_extraction\": \"Therefore, the answer is\", \n",
    "    'answer':\"\", \n",
    "    'cot': \"\", \n",
    "    'reflection_prompt':\"The Answer was given by a language model and may be incorrect. Multiple answers may be valid but only one answer is the best correct answer. If you give more than one answer, it will be evaluated as wrong. Revise, then pick the one answer that is most likely for this question.\",\n",
    "    'reflect_answer_extraction':'Based on the reflection, the answer is therefore',\n",
    "    'model_name':\"gpt-4\"\n",
    "}\n",
    "ts_hard.metareason_flexible(chain=reflect_overall_chain,input_dict=input_dict)\n",
    "ts_hard.dump(\"third_prompt_gpt_4.json\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "e57afdbd867e4941888c6e211b98bfe4",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/12 [00:00<?, ?ex/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "f3a6948ed3a74d4abaf67c37d9c761f8",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/7 [00:00<?, ?ex/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "1ef81aec4b4d43a8af40b6dbcc0874f8",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/11 [00:00<?, ?ex/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "4e7f708d1208466d8a5700b8cc8c9ae9",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/21 [00:00<?, ?ex/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "726ab4821ffd4d238c52e01e2d0377c1",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/20 [00:00<?, ?ex/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "{'commonsense_qa': {'validation': {'accuracy': {'': {'_The Answer was given by a language model and may be incorrect. Multiple answers may be valid but only one answer is the best correct answer. If you give more than one answer, it will be evaluated as wrong. Revise, then pick the one answer that is most likely for this question._Based on the reflection, the answer is therefore': 0.75},\n",
       "    'text-davinci-003': {'None_kojima-01_kojima-A-E': 0.0}}}},\n",
       " 'worldtree': {'test': {'accuracy': {'': {'_The Answer was given by a language model and may be incorrect. Multiple answers may be valid but only one answer is the best correct answer. If you give more than one answer, it will be evaluated as wrong. Revise, then pick the one answer that is most likely for this question._Based on the reflection, the answer is therefore': 0.857143},\n",
       "    'text-davinci-003': {'None_kojima-01_kojima-A-D': 0.0}}}},\n",
       " 'strategy_qa': {'train': {'accuracy': {'': {'_The Answer was given by a language model and may be incorrect. Multiple answers may be valid but only one answer is the best correct answer. If you give more than one answer, it will be evaluated as wrong. Revise, then pick the one answer that is most likely for this question._Based on the reflection, the answer is therefore': 0.545455},\n",
       "    'text-davinci-003': {'None_kojima-01_kojima-yes-no': 0.0}}}},\n",
       " 'open_book_qa': {'test': {'accuracy': {'': {'_The Answer was given by a language model and may be incorrect. Multiple answers may be valid but only one answer is the best correct answer. If you give more than one answer, it will be evaluated as wrong. Revise, then pick the one answer that is most likely for this question._Based on the reflection, the answer is therefore': 0.714286},\n",
       "    'text-davinci-003': {'None_kojima-01_kojima-A-D': 0.0}}}},\n",
       " 'med_qa': {'test': {'accuracy': {'': {'_The Answer was given by a language model and may be incorrect. Multiple answers may be valid but only one answer is the best correct answer. If you give more than one answer, it will be evaluated as wrong. Revise, then pick the one answer that is most likely for this question._Based on the reflection, the answer is therefore': 0.5},\n",
       "    'text-davinci-003': {'None_kojima-01_kojima-A-E': 0.0}}}}}"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ts_hard.evaluate()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "6b7228181175465ca31bc0e4996ffde3",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Creating json from Arrow format:   0%|          | 0/1 [00:00<?, ?ba/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "5cfd22769be34501bc926f4299379b36",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Creating json from Arrow format:   0%|          | 0/1 [00:00<?, ?ba/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "0bee1430408249be9c022774d0572e2f",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Creating json from Arrow format:   0%|          | 0/1 [00:00<?, ?ba/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "3fbeec8e627f4dd99d4118364b623075",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Creating json from Arrow format:   0%|          | 0/1 [00:00<?, ?ba/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "701a9298745f4aa58ed7f50f7f4ff089",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Creating json from Arrow format:   0%|          | 0/1 [00:00<?, ?ba/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "ts_hard.dump(\"third_evaluated\")"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Da Vinci on Da Vinci"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "ts_hard = Collection.from_json('ts_hard_v1.json')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#check answer extraction\n",
    "input_dict = {\n",
    "    \"cot_trigger\": \"Answer: Let's think step by step.\",\n",
    "    \"answer_extraction\": \"Therefore, the answer is\", \n",
    "    'answer':\"\", \n",
    "    'cot': \"\", \n",
    "    'reflection_prompt':\"The goal is to correct the Answer (generated by a language model) if needed, let's think step by step\" ,\n",
    "    'reflect_answer_extraction':'Based on the reflection, the answer is therefore',\n",
    "    'model_name':\"gpt-3.5-turbo\"\n",
    "}\n",
    "ts_hard.metareason_flexible(chain=reflect_overall_chain,input_dict=input_dict)\n",
    "ts_hard.evaluate()\n",
    "ts_hard.dump(\"first_prompt_davinci.json\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "6f0420d4c8d84c98a588f9976da36a35",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/12 [00:00<?, ?ex/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "95678f13b5d1482abf40fa65df1107d6",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/7 [00:00<?, ?ex/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "df6d9a241ef24bddad299c4a31726b45",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/11 [00:00<?, ?ex/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "cb0e1bdae214487a9c9e24a4ecd88185",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/21 [00:00<?, ?ex/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "1db6ecbd07554976bc1bba8b47501b74",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/20 [00:00<?, ?ex/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "{'commonsense_qa': {'validation': {'accuracy': {'': {\"_The goal is to correct the Answer (generated by a language model) if needed, let's think step by step_Based on the reflection, the answer is therefore\": 0.0},\n",
       "    'text-davinci-003': {'None_kojima-01_kojima-A-E': 0.0}}}},\n",
       " 'worldtree': {'test': {'accuracy': {'': {\"_The goal is to correct the Answer (generated by a language model) if needed, let's think step by step_Based on the reflection, the answer is therefore\": 0.0},\n",
       "    'text-davinci-003': {'None_kojima-01_kojima-A-D': 0.0}}}},\n",
       " 'strategy_qa': {'train': {'accuracy': {'': {\"_The goal is to correct the Answer (generated by a language model) if needed, let's think step by step_Based on the reflection, the answer is therefore\": 0.0},\n",
       "    'text-davinci-003': {'None_kojima-01_kojima-yes-no': 0.0}}}},\n",
       " 'open_book_qa': {'test': {'accuracy': {'': {\"_The goal is to correct the Answer (generated by a language model) if needed, let's think step by step_Based on the reflection, the answer is therefore\": 0.0},\n",
       "    'text-davinci-003': {'None_kojima-01_kojima-A-D': 0.0}}}},\n",
       " 'med_qa': {'test': {'accuracy': {'': {\"_The goal is to correct the Answer (generated by a language model) if needed, let's think step by step_Based on the reflection, the answer is therefore\": 0.0},\n",
       "    'text-davinci-003': {'None_kojima-01_kojima-A-E': 0.0}}}}}"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ts_hard.evaluate()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "ts_hard_2 = Collection.from_json('ts_hard_v1.json')\n",
    "input_dict = {\n",
    "    \"cot_trigger\": \"Answer: Let's think step by step.\",\n",
    "    \"answer_extraction\": \"Therefore, the answer is\", \n",
    "    'answer':\"\", \n",
    "    'cot': \"\", \n",
    "    'reflection_prompt':\"The question may try to trick the respondent and is designed to have one best answer. With this information, revise the previous answer made by a language model\" ,\n",
    "    'reflect_answer_extraction':'Based on the reflection, the answer is therefore',\n",
    "    'model_name':\"text-davinci-003\"\n",
    "}\n",
    "ts_hard_2.metareason_flexible(chain=reflect_overall_chain,input_dict=input_dict)\n",
    "ts_hard_2.evaluate()\n",
    "ts_hard_2.dump(\"second_prompt_davinci.json\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "4e62132970cc4262bcc5edb95aaea4e5",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/12 [00:00<?, ?ex/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "558a4985f0bd4d99958e9627a839560b",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/7 [00:00<?, ?ex/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "df829352204341d3a940ad6891ba4db0",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/11 [00:00<?, ?ex/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "837a6322169742f88b322a5f4f29a22d",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/21 [00:00<?, ?ex/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "74d1d428a2ec40ccadee3186d76dd443",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/20 [00:00<?, ?ex/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "{'commonsense_qa': {'validation': {'accuracy': {'': {'_The question may try to trick the respondent and is designed to have one best answer. With this information, revise the previous answer made by a language model_Based on the reflection, the answer is therefore': 0.0},\n",
       "    'text-davinci-003': {'None_kojima-01_kojima-A-E': 0.0}}}},\n",
       " 'worldtree': {'test': {'accuracy': {'': {'_The question may try to trick the respondent and is designed to have one best answer. With this information, revise the previous answer made by a language model_Based on the reflection, the answer is therefore': 0.0},\n",
       "    'text-davinci-003': {'None_kojima-01_kojima-A-D': 0.0}}}},\n",
       " 'strategy_qa': {'train': {'accuracy': {'': {'_The question may try to trick the respondent and is designed to have one best answer. With this information, revise the previous answer made by a language model_Based on the reflection, the answer is therefore': 0.0},\n",
       "    'text-davinci-003': {'None_kojima-01_kojima-yes-no': 0.0}}}},\n",
       " 'open_book_qa': {'test': {'accuracy': {'': {'_The question may try to trick the respondent and is designed to have one best answer. With this information, revise the previous answer made by a language model_Based on the reflection, the answer is therefore': 0.047619},\n",
       "    'text-davinci-003': {'None_kojima-01_kojima-A-D': 0.0}}}},\n",
       " 'med_qa': {'test': {'accuracy': {'': {'_The question may try to trick the respondent and is designed to have one best answer. With this information, revise the previous answer made by a language model_Based on the reflection, the answer is therefore': 0.0},\n",
       "    'text-davinci-003': {'None_kojima-01_kojima-A-E': 0.0}}}}}"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ts_hard_2.evaluate()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "ts_hard_3 = Collection.from_json('ts_hard_v1.json')\n",
    "input_dict = {\n",
    "    \"cot_trigger\": \"Answer: Let's think step by step.\",\n",
    "    \"answer_extraction\": \"Therefore, the answer is\", \n",
    "    'answer':\"\", \n",
    "    'cot': \"\", \n",
    "    'reflection_prompt':\"The Answer was given by a language model and may be incorrect. Multiple answers may be valid but only one answer is the best correct answer. If you give more than one answer, it will be evaluated as wrong. Revise, then pick the one answer that is most likely for this question.\",\n",
    "    'reflect_answer_extraction':'Based on the reflection, the answer is therefore',\n",
    "    'model_name':\"text-davinci-003\"\n",
    "}\n",
    "ts_hard_3.metareason_flexible(chain=reflect_overall_chain,input_dict=input_dict)\n",
    "ts_hard_3.evaluate()\n",
    "ts_hard_3.dump(\"third_prompt_davinci.json\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "bfbb8d33d2bc4c5bbb708933e8a211db",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/12 [00:00<?, ?ex/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "1157a4f536fa46c886390ca83aa21dcc",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/7 [00:00<?, ?ex/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "023faeda9baf4ad9bcd26b0ed4783858",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/11 [00:00<?, ?ex/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "1b5cd121b0c246258da40582c390929b",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/21 [00:00<?, ?ex/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "23d693ec40db423cacc66219853661ad",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/20 [00:00<?, ?ex/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "{'commonsense_qa': {'validation': {'accuracy': {'': {'_The Answer was given by a language model and may be incorrect. Multiple answers may be valid but only one answer is the best correct answer. If you give more than one answer, it will be evaluated as wrong. Revise, then pick the one answer that is most likely for this question._Based on the reflection, the answer is therefore': 0.083333},\n",
       "    'text-davinci-003': {'None_kojima-01_kojima-A-E': 0.0}}}},\n",
       " 'worldtree': {'test': {'accuracy': {'': {'_The Answer was given by a language model and may be incorrect. Multiple answers may be valid but only one answer is the best correct answer. If you give more than one answer, it will be evaluated as wrong. Revise, then pick the one answer that is most likely for this question._Based on the reflection, the answer is therefore': 0.0},\n",
       "    'text-davinci-003': {'None_kojima-01_kojima-A-D': 0.0}}}},\n",
       " 'strategy_qa': {'train': {'accuracy': {'': {'_The Answer was given by a language model and may be incorrect. Multiple answers may be valid but only one answer is the best correct answer. If you give more than one answer, it will be evaluated as wrong. Revise, then pick the one answer that is most likely for this question._Based on the reflection, the answer is therefore': 0.181818},\n",
       "    'text-davinci-003': {'None_kojima-01_kojima-yes-no': 0.0}}}},\n",
       " 'open_book_qa': {'test': {'accuracy': {'': {'_The Answer was given by a language model and may be incorrect. Multiple answers may be valid but only one answer is the best correct answer. If you give more than one answer, it will be evaluated as wrong. Revise, then pick the one answer that is most likely for this question._Based on the reflection, the answer is therefore': 0.095238},\n",
       "    'text-davinci-003': {'None_kojima-01_kojima-A-D': 0.0}}}},\n",
       " 'med_qa': {'test': {'accuracy': {'': {'_The Answer was given by a language model and may be incorrect. Multiple answers may be valid but only one answer is the best correct answer. If you give more than one answer, it will be evaluated as wrong. Revise, then pick the one answer that is most likely for this question._Based on the reflection, the answer is therefore': 0.0},\n",
       "    'text-davinci-003': {'None_kojima-01_kojima-A-E': 0.0}}}}}"
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ts_hard_3.evaluate()"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Binary classification"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "#llm = ChatOpenAI(temperature=.0,model_name=\"gpt-3.5-turbo\") #ADA #for chat: gpt-3.5-turbo\n",
    "#llm = ChatOpenAI(temperature=.0,model_name=\"gpt-4\") \n",
    "llm = OpenAI(temperature=.0,model_name=\"text-davinci-003\") \n",
    "\n",
    "\"\"\"answer extraction chain\"\"\"\n",
    "\n",
    "extraction_template = \"\"\"{instruction}\n",
    "\n",
    "Question: {question}\n",
    "Answer_choices: {answer_choices}\n",
    "\n",
    "Reasoning: {cot_trigger}{cot}\n",
    "{answer_extraction}\n",
    "\"\"\"\n",
    "\n",
    "prompt_template = PromptTemplate(input_variables=[\"instruction\",\"question\",\"answer_choices\",\"cot_trigger\",\"cot\",\"answer_extraction\"], template=extraction_template)\n",
    "answer_chain = LLMChain(llm=llm, prompt=prompt_template,output_key=\"predicted_answer\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [],
   "source": [
    "#{instruction}\n",
    "llm = OpenAI(temperature=.0,model_name=\"text-davinci-003\") \n",
    "reflect_extraction_template = \"\"\"\n",
    "\n",
    "    Question: {question}\n",
    "    Answer_choices: {answer_choices}\n",
    "\n",
    "    {cot_trigger} {cot}\n",
    "    {answer_extraction} {answer}\n",
    "    \n",
    "    Reflection: {reflect_answer_extraction}\n",
    "    \"\"\"\n",
    "    #Get reflection\n",
    "reflect_ans_prompt_template = PromptTemplate(input_variables=[\"question\",\"answer_choices\",\"cot_trigger\",\"cot\",\"answer_extraction\",'answer','reflect_answer_extraction'], template=reflect_extraction_template)\n",
    "single_reflect_answer_chain = LLMChain(llm=llm, prompt=reflect_ans_prompt_template,output_key=\"predicted_answer\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/anaconda3/lib/python3.9/site-packages/scipy/__init__.py:146: UserWarning: A NumPy version >=1.16.5 and <1.23.0 is required for this version of SciPy (detected version 1.24.2\n",
      "  warnings.warn(f\"A NumPy version >={np_minversion} and <{np_maxversion}\"\n",
      "[nltk_data] Downloading package punkt to\n",
      "[nltk_data]     /Users/robertpraas/nltk_data...\n",
      "[nltk_data]   Package punkt is already up-to-date!\n"
     ]
    }
   ],
   "source": [
    "ts_hard = Collection.from_json('ts_hard_v1.json')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "| Name           | Train   | Valid   | Test   |\n",
       "|----------------|---------|---------|--------|\n",
       "| commonsense_qa | -       | 4       | -      |\n",
       "| med_qa         | -       | -       | 1      |\n",
       "| open_book_qa   | -       | -       | 12     |\n",
       "| strategy_qa    | 3       | -       | -      |\n",
       "| worldtree      | -       | -       | 1      |\n",
       "\n",
       "Not loaded: ['aqua', 'asdiv', 'entailment_bank', 'gsm8k', 'mawps', 'medmc_qa', 'pubmed_qa', 'qed', 'svamp']"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ts_hard"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#connect answer\n",
    "\"\"\"\n",
    "\n",
    "First: \"answer_extraction\": \"The Reasoning is done by a language model and may contain bias, noise or factual inconsistencies. If the reasoning is correct, answer with 'True'. If there is suspicion that the Reasoning is incorrect, answer with 'False'\" \n",
    "Second: \"answer_extraction\": \"You are a corrector model that needs to judge whether the provided Reasoning, which was done by a language model, is factually correct. If you think there might be a problem with the Reasoning, state 'False.' If you believe nothing is wrong with the reasoning, state 'True'.\"\n",
    "Third: \"You will provide a Critique of the Reasoning, highlighting the limitations, inaccuracies, or areas that need improvement or expansion, while providing guidance on how to address these issues in a response. Start with 'False' if the Reasoning leads to the wrong answer, else start with 'True'.\" \n",
    "Fourth: \"A corrector model indicates the Reasoning leads to the wrong answer to the question. Provide a better answer.\"\n",
    "\"\"\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Generating commonsense_qa...\n",
      "Generating med_qa...\n",
      "Generating open_book_qa...\n",
      "Generating strategy_qa...\n",
      "Generating worldtree...\n"
     ]
    }
   ],
   "source": [
    "ts_hard = Collection.from_json(\"ts_hard_chatgpt.json\")\n",
    "\n",
    "\"\"\"Extract script: Assumes there are CoTs in the dataset already\"\"\"\n",
    "\n",
    "input_dict = {\n",
    "    \"instruction\": \"\",\n",
    "    \"cot_trigger\": \"Answer: Let's think step by step.\",\n",
    "    \"answer_extraction\":  \"A corrector model indicates the Reasoning leads to the wrong answer to the question. Provide a better answer.\"\n",
    "}\n",
    "\n",
    "ts_hard.extract_flexible(chain=answer_chain,input_dict=input_dict)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "d99a990b2fe8498d92ee79b7cf6d7bf4",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Creating json from Arrow format:   0%|          | 0/1 [00:00<?, ?ba/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "28cb532de8d144edb882315ba5c88bea",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Creating json from Arrow format:   0%|          | 0/1 [00:00<?, ?ba/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "dd79e38e7a264afe809584246d7a818b",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Creating json from Arrow format:   0%|          | 0/1 [00:00<?, ?ba/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "3ca1467510534ac9b76f05bb0aada12f",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Creating json from Arrow format:   0%|          | 0/1 [00:00<?, ?ba/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "60d6d8fe795b4c81b2d0af25f1394881",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Creating json from Arrow format:   0%|          | 0/1 [00:00<?, ?ba/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "ts_hard.dump(\"chatgpt_checks_chatgpt_4\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "66b7f92770d147e0b96c47c1ecc18a15",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/4 [00:00<?, ?ex/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "15d8dcb30bf04e6b8bb693ee6fd72a22",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/1 [00:00<?, ?ex/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "3acd1c654d904e0886853554d60ca49c",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/12 [00:00<?, ?ex/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "86e69296cded466ea8fc3a27033f8f7c",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/3 [00:00<?, ?ex/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "2b006b6f7ffb455996edcbbe46ce789f",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/1 [00:00<?, ?ex/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "{'commonsense_qa': {'validation': {'accuracy': {'gpt-3.5-turbo': {'None_kojima-01_A corrector model indicates the Reasoning leads to the wrong answer to the question. Provide a better answer.': 0.25,\n",
       "     'None_kojima-01_kojima-A-E': 0.0}}}},\n",
       " 'med_qa': {'test': {'accuracy': {'gpt-3.5-turbo': {'None_kojima-01_A corrector model indicates the Reasoning leads to the wrong answer to the question. Provide a better answer.': 0.0,\n",
       "     'None_kojima-01_kojima-A-E': 0.0}}}},\n",
       " 'open_book_qa': {'test': {'accuracy': {'gpt-3.5-turbo': {'None_kojima-01_A corrector model indicates the Reasoning leads to the wrong answer to the question. Provide a better answer.': 0.25,\n",
       "     'None_kojima-01_kojima-A-D': 0.0}}}},\n",
       " 'strategy_qa': {'train': {'accuracy': {'gpt-3.5-turbo': {'None_kojima-01_A corrector model indicates the Reasoning leads to the wrong answer to the question. Provide a better answer.': 0.0,\n",
       "     'None_kojima-01_kojima-yes-no': 0.0}}}},\n",
       " 'worldtree': {'test': {'accuracy': {'gpt-3.5-turbo': {'None_kojima-01_A corrector model indicates the Reasoning leads to the wrong answer to the question. Provide a better answer.': 0.0,\n",
       "     'None_kojima-01_kojima-A-D': 0.0}}}}}"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ts_hard.evaluate()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Correct answer: C) annually.\n",
      "\n",
      "Reasoning: An equinox occurs twice a year, in March and September, when the tilt of the Earth's axis is inclined neither away from nor towards the Sun, resulting in a nearly equal amount of daylight and darkness at all latitudes. Therefore, a person has a chance to experience an equinox once a year.\n",
      "A better answer would be:\n",
      "\n",
      "Answer: A large cargo ship or a freight train.\n",
      "\n",
      "As mentioned in the reasoning, these modes of transportation are designed to carry heavy loads and have a high total weight, making them the transportation modes with the most mass.\n",
      "Correct answer: A) The color of my hair. \n",
      "\n",
      "Reasoning: Over a period of time, the weather can cause changes in the amount of sunlight and UV radiation that a person is exposed to, which can lead to changes in hair color. This is especially true for people who spend a lot of time outdoors, as the sun can bleach or darken hair over time. While the weather can also affect the performance of electronic devices like computers, it is less likely to cause significant changes in the sound they make over time.\n",
      "Correct answer: B) the moon is brighter than stars.\n",
      "\n",
      "Reasoning: The question asks about instances where light from further away may appear less bright than closer sources. In this case, the moon is further away than stars, but it appears brighter because it reflects sunlight. The moon is also closer to Earth than most stars, which makes it appear larger and brighter in the night sky. Therefore, the correct answer is B) the moon is brighter than stars.\n",
      "Correct answer: B) water and oil.\n",
      "\n",
      "The reasoning provided is correct, and the answer is also correct. Water and oil do not naturally mix together, but an emulsifier such as soap can be used to combine them. This is a common experiment in science classes and can demonstrate the concept of emulsification and the importance of using a substance to help mix two substances that do not naturally combine. Therefore, the correct answer is B) water and oil.\n",
      "Better answer: C) significant supplies accumulated prior. While humans predate the formation of fossil fuels, there were significant supplies of fossil fuels already accumulated in the ground by the time humans evolved. These supplies were formed from the remains of plants and animals that lived millions of years before humans, and were buried and transformed into fossil fuels over time. Therefore, while humans did not create fossil fuels, they have been able to extract and use them for energy since discovering their existence.\n",
      "Correct answer: None of the options guarantee rain.\n",
      "\n",
      "Reasoning: While cumulus clouds are often associated with rain showers, their presence does not guarantee rain. Similarly, the presence of cirrus clouds, hail stones, or direct sunshine does not guarantee the absence of rain. Rain is a complex weather phenomenon that depends on a variety of factors, including temperature, humidity, atmospheric pressure, and wind patterns. Therefore, none of the options can be considered a guarantee of rain.\n",
      "Correct answer: D) a switch\n",
      "\n",
      "Reasoning: To turn on an electrical device, we need a mechanism that can complete the circuit and supply power to the device. A switch is a device that can do this by connecting or disconnecting the circuit. When the switch is turned on, it completes the circuit and allows the flow of electricity to power the device. Therefore, the correct answer is D) a switch.\n",
      "Correct answer: A) the Sahara\n",
      "\n",
      "Reasoning: A warm-weather organism is an organism that thrives in warm temperatures and cannot survive in colder climates. The Sahara is a hot and dry desert, which means it could be a possible habitat for warm-weather organisms. The mountains can have varying temperatures depending on the altitude, so it's not a guaranteed habitat for warm-weather organisms. The ocean can have warm and cold temperatures depending on the location, but there are definitely warm-weather organisms that live in tropical waters. The sewers are not a natural habitat for any organism, let alone warm-weather organisms. Therefore, the correct answer is A) the Sahara.\n",
      "Correct answer: Without additional information, it is difficult to determine the exact cause for the decrease in brush in the park. Other possible causes could include natural disasters such as wildfires or disease outbreaks among the vegetation. It is important to conduct further investigation and analysis to determine the root cause of the decrease in brush.\n",
      "Better answer: The reasoning is incorrect. The question does not provide enough context to determine what unit of measurement the balance result will be in. It is possible that the balance result could be in any of the units listed (kilowatts, kilobytes, kilograms, or kilometers) depending on the situation. Without more information, it is impossible to determine the correct answer.\n",
      "Correct answer: B) Trees\n",
      "\n",
      "The reasoning is correct, but the answer is incomplete. Camouflage can indeed be used by animals for hunting, and trees are a common example of a habitat where animals use camouflage to blend in and ambush prey. However, there are other habitats where animals use camouflage for hunting, such as grasslands, deserts, and even underwater environments. Therefore, the answer should be expanded to include these other habitats as well.\n",
      "Better answer: The question of whether a pound sterling is valuable or not is subjective and depends on the context. In the United Kingdom, where it is the official currency, it is valuable because it is widely accepted and can be used to purchase goods and services. However, its value fluctuates against other currencies depending on various economic and political factors. Therefore, the answer is not a simple true or false, but rather it depends on the context and the comparison being made.\n",
      "Correct answer: It is impossible to determine an answer to this question as it is nonsensical and lacks context. The Toyota Hilux and Mr. Ed are unrelated entities with different abilities and purposes, making it impossible to compare them in any meaningful way.\n",
      "Correct answer: B) False, the concert would not be audible outside of the space station, but it would be audible inside the station.\n",
      "Correct answer: D) carpet.\n",
      "\n",
      "The reasoning is correct in eliminating options A, C, and E. Option B is a possibility, but it's too general. However, the correct answer is D) carpet, as it is a surface that cats can dig their claws into and provides traction. Without a carpet, the kitten would have nothing to grip onto and would slide across the floor. Therefore, the kitten slid across the carpet.\n",
      "A better answer would be E) tell a story. Before nap time, a kindergarten teacher often reads a story or sings a lullaby to help children relax and feel comfortable. This can also help them transition from playtime to nap time and create a calming environment. Encouragement may also be a part of the process, but telling a story is a more specific and common activity that a kindergarten teacher would do before nap time.\n",
      "Better answer: The country with the most fast food restaurants is the United States of America. According to a 2021 report by QSR Magazine, the top 50 largest fast food chains in the world are dominated by American brands, with McDonald's being the largest chain globally. The report also states that the United States has the highest number of fast food restaurants per capita compared to any other country. Therefore, option D, America, is the correct answer.\n",
      "Better answer: It is difficult to determine where Billy might have moved to based on the given information. However, some possible options could be another large city with a different focus, such as a cultural or economic center, or a smaller city or town that offers a different lifestyle. It is also possible that Billy moved to a different country altogether. Without more information, it is impossible to determine a definitive answer.\n",
      "Correct answer: C) Spider bite.\n",
      "\n",
      "The patient's symptoms, including fever, night sweats, lethargy, and low blood pressure, are concerning for sepsis, which can be caused by a bacterial or fungal infection. However, the swollen bite marks on his right hand with red streaks extending to his elbow suggest a venomous spider bite, which can also cause systemic symptoms and lead to sepsis.\n",
      "\n",
      "In addition, the patient's history of sickle cell disease puts him at increased risk for complications from a spider bite, including tissue damage and organ failure.\n",
      "\n",
      "Therefore, the most concerning bite in this patient would be a spider bite, and urgent treatment with antivenom and supportive care is necessary.\n",
      "Correct answer: None of the events listed occur once per day. The Moon rising and setting occurs once per day, but it is not specific to the Moon. The Moon cycles through its phases once per lunar month, and a solar eclipse occurs only a few times per year. Therefore, there is no event related to the Moon that occurs once per day.\n",
      "Correct:1\n",
      "Incorrect:1\n",
      "Undecided:1\n"
     ]
    }
   ],
   "source": [
    "#fourth\n",
    "import re\n",
    "data = [('open_book_qa','test'),('strategy_qa','train'),('commonsense_qa','validation'),('med_qa','test'),('worldtree','test')]\n",
    "\n",
    "correct = 0\n",
    "incorrect = 0\n",
    "undecided = 0\n",
    "unclassified = 0\n",
    "\n",
    "for pair in data:\n",
    "    #df_pandas = .to_pandas()\n",
    "    for row in ts_hard[pair[0]][pair[1]]:\n",
    "        text = row['generated_cot'][0]['answers'][1]['answer']\n",
    "        print(text)\n",
    "        true = bool(re.search(r\"(?i)True\", text))\n",
    "        false = bool(re.search(r\"(?i)False\", text))\n",
    "        if true & false:\n",
    "            undecided +=1 \n",
    "        elif true:\n",
    "            correct+=1\n",
    "        elif false:\n",
    "            incorrect+=1\n",
    "        else:\n",
    "            unclassified+=1\n",
    "\n",
    "print(f\"Correct:{correct}\")\n",
    "print(f\"Incorrect:{incorrect}\")\n",
    "print(f\"Undecided:{undecided}\")\n",
    "print(f\"Unclassified:{unclassified}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "metadata": {},
   "outputs": [],
   "source": [
    "ts_hard = Collection.from_json(\"../notebooks/chatgpt_checks_chatgpt_1.json\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "True\n",
      "True\n",
      "True\n",
      "True\n",
      "True\n",
      "True\n",
      "True\n",
      "True\n",
      "True\n",
      "True\n",
      "True\n",
      "True\n",
      "True\n",
      "False\n",
      "True.\n",
      "True\n",
      "True\n",
      "True\n",
      "True\n",
      "True\n",
      "False\n",
      "Correct:19\n",
      "Incorrect:2\n",
      "Undecided:0\n",
      "Unclassified:0\n"
     ]
    }
   ],
   "source": [
    "#1st\n",
    "import re\n",
    "data = [('open_book_qa','test'),('strategy_qa','train'),('commonsense_qa','validation'),('med_qa','test'),('worldtree','test')]\n",
    "\n",
    "correct = 0\n",
    "incorrect = 0\n",
    "undecided = 0\n",
    "unclassified = 0\n",
    "\n",
    "for pair in data:\n",
    "    #df_pandas = .to_pandas()\n",
    "    for row in ts_hard[pair[0]][pair[1]]:\n",
    "        text = row['generated_cot'][0]['answers'][1]['answer']\n",
    "        print(text)\n",
    "        true = bool(re.search(r\"(?i)True\", text))\n",
    "        false = bool(re.search(r\"(?i)False\", text))\n",
    "        if true & false:\n",
    "            undecided +=1 \n",
    "        elif true:\n",
    "            correct+=1\n",
    "        elif false:\n",
    "            incorrect+=1\n",
    "        else:\n",
    "            unclassified+=1\n",
    "\n",
    "print(f\"Correct:{correct}\")\n",
    "print(f\"Incorrect:{incorrect}\")\n",
    "print(f\"Undecided:{undecided}\")\n",
    "print(f\"Unclassified:{unclassified}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "True.\n",
      "True.\n",
      "True.\n",
      "True.\n",
      "True.\n",
      "True.\n",
      "True.\n",
      "True.\n",
      "True.\n",
      "True.\n",
      "True.\n",
      "True.\n",
      "True.\n",
      "True.\n",
      "True.\n",
      "True.\n",
      "True.\n",
      "True.\n",
      "True.\n",
      "True.\n",
      "False. The Moon cycles through its phases occur once per lunar month, not once per day.\n",
      "Correct:20\n",
      "Incorrect:1\n",
      "Undecided:0\n",
      "Unclassified:0\n"
     ]
    }
   ],
   "source": [
    "#2nd\n",
    "import re\n",
    "data = [('open_book_qa','test'),('strategy_qa','train'),('commonsense_qa','validation'),('med_qa','test'),('worldtree','test')]\n",
    "\n",
    "correct = 0\n",
    "incorrect = 0\n",
    "undecided = 0\n",
    "unclassified = 0\n",
    "\n",
    "for pair in data:\n",
    "    #df_pandas = .to_pandas()\n",
    "    for row in ts_hard[pair[0]][pair[1]]:\n",
    "        text = row['generated_cot'][0]['answers'][1]['answer']\n",
    "        print(text)\n",
    "        true = bool(re.search(r\"(?i)True\", text))\n",
    "        false = bool(re.search(r\"(?i)False\", text))\n",
    "        if true & false:\n",
    "            undecided +=1 \n",
    "        elif true:\n",
    "            correct+=1\n",
    "        elif false:\n",
    "            incorrect+=1\n",
    "        else:\n",
    "            unclassified+=1\n",
    "\n",
    "print(f\"Correct:{correct}\")\n",
    "print(f\"Incorrect:{incorrect}\")\n",
    "print(f\"Undecided:{undecided}\")\n",
    "print(f\"Unclassified:{unclassified}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "True. The reasoning is accurate and correctly identifies that an equinox occurs twice a year, resulting in nearly equal amounts of daylight and darkness at all latitudes. The answer choice C) annually is the correct choice. No limitations, inaccuracies, or areas needing improvement or expansion were identified in the reasoning.\n",
      "True: The reasoning is accurate in explaining the concept of mass and transportation modes. However, the options provided in the question are limited and do not include all possible transportation modes. The correct answer, as mentioned in the reasoning, would be a large cargo ship or a freight train. To improve the question, the options should be expanded to include all possible transportation modes that could have the most mass. Additionally, the question could be more specific about the type of transportation being referred to, such as passenger transportation or cargo transportation.\n",
      "True. The reasoning is accurate and correctly eliminates options A, B, and C as unlikely or impossible. However, it could be expanded by providing more specific examples of how weather can affect the sound a computer makes, such as changes in humidity causing static or changes in temperature affecting the performance of internal components. Additionally, it could be helpful to mention other ways in which weather can affect our environment, such as changes in precipitation, wind, or temperature, which can all have significant impacts on our daily lives.\n",
      "True: The reasoning correctly identifies that option C is the answer to the question. However, the explanation could be improved by providing more context on why the moon appears brighter than a floodlight despite being further away. This could include discussing the reflective properties of the moon's surface and how it reflects sunlight, as well as the fact that floodlights are designed to be directional and focused, whereas the moon's light is diffused. Additionally, the reasoning could benefit from explaining why options A and D are incorrect, as this would help to eliminate confusion for those who may not understand why these options are not relevant to the question.\n",
      "True: The reasoning is accurate and provides a logical explanation for why option B, water and oil, is the correct answer. However, it could be expanded by providing more information on how an emulsifier works and why it is necessary to mix oil and water. Additionally, it may be helpful to mention other examples of emulsifiers and how they are used in everyday life.\n",
      "True. The reasoning is accurate and provides a correct answer to the question. However, it could be expanded by mentioning that while humans predate fossil fuel formation, there were still significant supplies of fossil fuels that accumulated prior to human existence. This would make option C) significant supplies accumulated prior, a partially correct answer. Additionally, it could be noted that while fossil fuels were not created by humans, their extraction and use have had a significant impact on human societies and the environment.\n",
      "True. The reasoning is accurate and provides a good explanation of how rain is formed and the role of different factors in its occurrence. However, it could be expanded by mentioning other types of clouds that can also produce rain, such as stratus and nimbus clouds. Additionally, it is important to note that the presence of cumulus clouds alone does not guarantee rain, as other factors such as temperature and humidity levels also play a role. Therefore, the answer could be improved by stating that cumulus clouds increase the likelihood of rain, but are not a guarantee.\n",
      "True. The reasoning is accurate and provides a clear explanation of the process of turning on an electrical device. However, it could be expanded by providing more examples of how a magnet can be used to turn on electrical devices, such as in magnetic switches or sensors. Additionally, it would be helpful to clarify that while a magnet can be used to activate a switch or sensor, it is not the only mechanism for turning on an electrical device. Other methods include pressing a button, flipping a switch, or using a remote control.\n",
      "True. The reasoning is accurate and provides a logical thought process to arrive at the correct answer. However, it could be expanded by providing examples of warm-weather organisms that live in each of the habitats mentioned. Additionally, it's important to note that warm-weather organisms can also be found in other habitats such as rainforests, deserts, and savannas.\n",
      "True. The reasoning is sound and correctly identifies option D as the most likely cause for the decrease in brush in the park. However, it would be helpful to consider other possible causes as well, such as natural disasters like wildfires or disease outbreaks that could affect the vegetation. Additionally, it would be useful to gather more information about the park, such as the history of human activity and any recent changes in management practices, to better understand the factors contributing to the decrease in brush.\n",
      "True: The reasoning is sound and leads to the correct answer. However, it is important to note that the question does not provide enough context to determine what is being measured on the balance. Therefore, while kilometers may be the most likely answer, it is not necessarily the only possible answer. Additionally, it would be helpful to have more information about the situation in order to make a more informed decision about which unit of measurement is most appropriate for the balance result.\n",
      "True. The reasoning is accurate and provides a logical explanation for why animals use camouflage for hunting. However, it could be improved by providing specific examples of animals that use camouflage while hunting in trees, such as the owl or the jaguar. Additionally, it would be helpful to clarify that while camouflage can be used for hunting, it can also be used for defense or to attract a mate.\n",
      "True. The reasoning is accurate in stating that the value of the pound sterling depends on the context and comparison being made. However, it could be expanded by acknowledging that the value of the pound sterling is also influenced by factors such as inflation, interest rates, and political stability. Additionally, the answer could be improved by providing examples of when the pound sterling has been particularly valuable or not valuable in certain contexts.\n",
      "False. While the reasoning correctly identifies the meaning of \"tip the scales,\" it assumes that \"Mr. Ed\" refers to the TV show character without any context or clarification. This could lead to confusion or misinterpretation of the question. Additionally, the reasoning does not provide any evidence or explanation for why the Toyota Hilux could not tip the scales against Mr. Ed if it were referring to a different entity. To improve the reasoning, it would be helpful to provide more context or clarification on who or what \"Mr. Ed\" refers to and to provide a more detailed explanation for why the Toyota Hilux could not tip the scales against that entity.\n",
      "True. The reasoning is correct in stating that sound cannot travel through the vacuum of space and therefore the concert would not be audible outside of the space station. However, the reasoning could be expanded to include the fact that even inside the space station, the sound would be greatly reduced due to the lack of air molecules to transmit the sound waves. Therefore, the sound would need to be amplified and transmitted through the station's speakers in order for it to be audible to the astronauts. Additionally, the acoustics of the space station may not be ideal for a concert, as the lack of gravity and confined space could affect the quality of the sound. These limitations could be addressed by using specialized equipment and designing the concert to take advantage of the unique environment of the space station.\n",
      "True. The reasoning is logical and correctly eliminates options that do not make sense. However, it could be improved by providing more context about the situation. For example, where was the kitten? Was it in a house or outside? What kind of flooring was present? These details could help narrow down the answer even further. Additionally, it would be helpful to explain why the other options are not likely answers. For example, why is a warm place not a possible answer? Providing more information and explanation would make the reasoning more thorough and convincing.\n",
      "True. The reasoning is accurate and provides a comprehensive list of tasks that a kindergarten teacher typically does before nap time. The answer choice B) encourage is the most appropriate as it aligns with the teacher's role in helping children settle down and feel comfortable before nap time. However, it could be expanded by mentioning other activities that a teacher may do, such as providing a snack or a drink, or allowing children to choose a stuffed animal or blanket to sleep with. Additionally, it is important to note that nap time routines may vary depending on the school or classroom, and some teachers may have different approaches or strategies.\n",
      "True. The reasoning is generally sound, but it could be improved by providing more specific data or sources to support the claim that big cities have the highest concentration of fast food restaurants. Additionally, it would be helpful to clarify that while America may have a high number of fast food restaurants, it is not necessarily the country with the most. To address these issues, the response could provide statistics or studies on the concentration of fast food restaurants in different countries or regions, and explain how factors such as population density and consumer demand contribute to the distribution of fast food chains.\n",
      "True. The reasoning is logical and correctly eliminates options A, B, C, and E. However, option D is not specific enough as it could refer to many cities around the world. To improve the reasoning, it could be expanded to consider factors such as Billy's job, lifestyle preferences, and cultural background, which could influence his choice of a new city. For example, if Billy worked in the tech industry, he might move to a city with a thriving tech scene, such as San Francisco or Seattle. Alternatively, if he preferred a more laid-back lifestyle, he might move to a smaller city with a slower pace of life, such as Portland or Austin. By considering these additional factors, the reasoning could provide a more accurate and nuanced answer.\n",
      "True. The reasoning is sound and takes into account the patient's medical history, symptoms, and physical examination findings. However, it would be helpful to consider other possible causes of the patient's symptoms, such as a bacterial or viral infection unrelated to a bite. Additionally, further diagnostic tests, such as blood cultures and imaging studies, may be necessary to confirm the diagnosis and guide treatment. Finally, it is important to address the patient's sickle cell disease and its potential complications, such as increased susceptibility to infections and sepsis.\n",
      "True. The reasoning is accurate and correctly identifies that the Moon cycles through its phases once per day. No limitations, inaccuracies, or areas needing improvement or expansion were identified.\n",
      "Correct:20\n",
      "Incorrect:1\n",
      "Undecided:0\n"
     ]
    }
   ],
   "source": [
    "#3rd        "
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## try with double call"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "input_dict = {\n",
    "    \"cot_trigger\": \"Answer: Let's think step by step.\",\n",
    "    \"answer_extraction\": \"The answer is therefore\", \n",
    "    'answer':\"\", \n",
    "    'cot': \"\", \n",
    "    'reflection_prompt':\"A corrector model indicates the Reasoning leads to the wrong answer to the question. Provide a better answer.\",\n",
    "    'reflect_answer_extraction':'Based on the reflection, what is the definite answer?'\n",
    "}\n",
    "ts_hard.metareason_flexible(chain=reflect_overall_chain,input_dict=input_dict)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "3b26545821a34c2d80c3ab9444b30338",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/4 [00:00<?, ?ex/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "c77b9c72f0914b9b8896fd0a931dc5ec",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/1 [00:00<?, ?ex/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "12ec58a81f4d4fe3b6d5ac7f5fd876d2",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/12 [00:00<?, ?ex/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "b4d201a22f8a4e309d405d534c96a2bd",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/3 [00:00<?, ?ex/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "4a6f07b73b38468580afb195b11b8906",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/1 [00:00<?, ?ex/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "{'commonsense_qa': {'validation': {'accuracy': {'': {'_A corrector model indicates the Reasoning leads to the wrong answer to the question. Provide a better answer._Based on the reflection, what is the definite answer?': 0.25},\n",
       "    'gpt-3.5-turbo': {'None_kojima-01_A corrector model indicates the Reasoning leads to the wrong answer to the question. Provide a better answer.': 0.25,\n",
       "     'None_kojima-01_kojima-A-E': 0.0}}}},\n",
       " 'med_qa': {'test': {'accuracy': {'': {'_A corrector model indicates the Reasoning leads to the wrong answer to the question. Provide a better answer._Based on the reflection, what is the definite answer?': 0.0},\n",
       "    'gpt-3.5-turbo': {'None_kojima-01_A corrector model indicates the Reasoning leads to the wrong answer to the question. Provide a better answer.': 0.0,\n",
       "     'None_kojima-01_kojima-A-E': 0.0}}}},\n",
       " 'open_book_qa': {'test': {'accuracy': {'': {'_A corrector model indicates the Reasoning leads to the wrong answer to the question. Provide a better answer._Based on the reflection, what is the definite answer?': 0.166667},\n",
       "    'gpt-3.5-turbo': {'None_kojima-01_A corrector model indicates the Reasoning leads to the wrong answer to the question. Provide a better answer.': 0.25,\n",
       "     'None_kojima-01_kojima-A-D': 0.0}}}},\n",
       " 'strategy_qa': {'train': {'accuracy': {'': {'_A corrector model indicates the Reasoning leads to the wrong answer to the question. Provide a better answer._Based on the reflection, what is the definite answer?': 0.0},\n",
       "    'gpt-3.5-turbo': {'None_kojima-01_A corrector model indicates the Reasoning leads to the wrong answer to the question. Provide a better answer.': 0.0,\n",
       "     'None_kojima-01_kojima-yes-no': 0.0}}}},\n",
       " 'worldtree': {'test': {'accuracy': {'': {'_A corrector model indicates the Reasoning leads to the wrong answer to the question. Provide a better answer._Based on the reflection, what is the definite answer?': 0.0},\n",
       "    'gpt-3.5-turbo': {'None_kojima-01_A corrector model indicates the Reasoning leads to the wrong answer to the question. Provide a better answer.': 0.0,\n",
       "     'None_kojima-01_kojima-A-D': 0.0}}}}}"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ts_hard.evaluate()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "69fae0db748c4987a2b2c5ee41ea4dfe",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Creating json from Arrow format:   0%|          | 0/1 [00:00<?, ?ba/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "2f04535f590b477fae08590791b58aaa",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Creating json from Arrow format:   0%|          | 0/1 [00:00<?, ?ba/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "3aea952b9c304b3b8fd48be3341300a8",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Creating json from Arrow format:   0%|          | 0/1 [00:00<?, ?ba/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "c268f034423b4438a60a44a7486166b4",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Creating json from Arrow format:   0%|          | 0/1 [00:00<?, ?ba/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "b46579eedb724832a3d4438b4abeaa5d",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Creating json from Arrow format:   0%|          | 0/1 [00:00<?, ?ba/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "ts_hard.dump(\"chatgpt_checks_chatgpt_double_4\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(ans_prompt_template.format(question=question,answer_choices=answer_choices,cot_trigger=cot_trigger,\n",
    "                                     cot=cot,answer_extraction=answer_extraction,\n",
    "                                     answer=answer,reflection_prompt=reflection_prompt,\n",
    "                                     reflection=reflection,reflect_answer_extraction=reflect_answer_extraction))"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Continue"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [],
   "source": [
    "multiple_ans_test = Collection.from_json(\"clean_chat_hard_multiple_ans.json\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "| Name           | Train   | Valid   | Test   |\n",
       "|----------------|---------|---------|--------|\n",
       "| commonsense_qa | -       | 7       | -      |\n",
       "| med_qa         | -       | -       | 3      |\n",
       "| open_book_qa   | -       | -       | 4      |\n",
       "\n",
       "Not loaded: ['aqua', 'asdiv', 'entailment_bank', 'gsm8k', 'mawps', 'medmc_qa', 'pubmed_qa', 'qed', 'strategy_qa', 'svamp', 'worldtree']"
      ]
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "multiple_ans_test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'\\n1:\"The above Answer to the Question contains multiple answers, however, only one answer is correct. The best answer is therefore\"\\n2:\"The above Answer to the Question contains multiple answers, however, only one answer is correct. Select one answer. The best answer is therefore\"\\n3:\"The multiple-choice question only contains one correct answer, giving multiple answers is wrong. From the Answer given, select the one best answer. The best answer is therefore\"\\n'"
      ]
     },
     "execution_count": 81,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "\"\"\"\n",
    "1:\"The above Answer to the Question contains multiple answers, however, only one answer is correct. The best answer is therefore\"\n",
    "2:\"The above Answer to the Question contains multiple answers, however, only one answer is correct. Select one answer. The best answer is therefore\"\n",
    "3:\"The multiple-choice question only contains one correct answer, giving multiple answers is wrong. From the Answer given, select the one best answer. The best answer is therefore\"\n",
    "\"\"\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Generating commonsense_qa...\n",
      "Generating med_qa...\n",
      "Generating open_book_qa...\n"
     ]
    }
   ],
   "source": [
    "\n",
    "\n",
    "\"\"\"Extract script: Assumes there are CoTs in the dataset already\"\"\"\n",
    "\n",
    "input_dict = {\n",
    "    \"instruction\": \"\",\n",
    "    \"cot_trigger\": \"Answer: Let's think step by step.\",\n",
    "    \"answer_extraction\":  \"The answer is therefore\",\n",
    "    'reflect_answer_extraction': \"The multiple-choice question only contains one correct answer, giving multiple answers is wrong. From the Answer given, select the one best answer. The best answer is therefore\"\n",
    "}\n",
    "\n",
    "multiple_ans_test.extract_flexible(chain=single_reflect_answer_chain,input_dict=input_dict)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "62791eadfb12481e834ee46f3ebebd33",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/7 [00:00<?, ?ex/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "be3905e58e124d0e9307e4a851c817bc",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/3 [00:00<?, ?ex/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "63bf1fd8c7a646da82ab3ebf70a9924a",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/4 [00:00<?, ?ex/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "{'commonsense_qa': {'validation': {'accuracy': {'': {\"_Answer: Let's think step by step._The answer is therefore\": 0.142857},\n",
       "    'gpt-3.5-turbo': {'None_kojima-01_kojima-A-E': 0.0}}}},\n",
       " 'med_qa': {'test': {'accuracy': {'': {\"_Answer: Let's think step by step._The answer is therefore\": 0.666667},\n",
       "    'gpt-3.5-turbo': {'None_kojima-01_kojima-A-E': 0.0}}}},\n",
       " 'open_book_qa': {'test': {'accuracy': {'': {\"_Answer: Let's think step by step._The answer is therefore\": 0.25},\n",
       "    'gpt-3.5-turbo': {'None_kojima-01_kojima-A-D': 0.0}}}}}"
      ]
     },
     "execution_count": 42,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#1\n",
    "multiple_ans_test.evaluate()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "62f57303f37c45a586b7863ae311698f",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/7 [00:00<?, ?ex/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "24e18d8e2207472e88e160119a9d537e",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/3 [00:00<?, ?ex/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "39e4f8b0ffcd41fe8afedc6d359dcdb4",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/4 [00:00<?, ?ex/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "{'commonsense_qa': {'validation': {'accuracy': {'': {\"_Answer: Let's think step by step._The answer is therefore\": 0.214286},\n",
       "    'gpt-3.5-turbo': {'None_kojima-01_kojima-A-E': 0.0}}}},\n",
       " 'med_qa': {'test': {'accuracy': {'': {\"_Answer: Let's think step by step._The answer is therefore\": 0.5},\n",
       "    'gpt-3.5-turbo': {'None_kojima-01_kojima-A-E': 0.0}}}},\n",
       " 'open_book_qa': {'test': {'accuracy': {'': {\"_Answer: Let's think step by step._The answer is therefore\": 0.25},\n",
       "    'gpt-3.5-turbo': {'None_kojima-01_kojima-A-D': 0.0}}}}}"
      ]
     },
     "execution_count": 45,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#2\n",
    "multiple_ans_test.evaluate()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "4947730105624742a488f2418432be52",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/7 [00:00<?, ?ex/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "576075e84b474061a450910635db563c",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/3 [00:00<?, ?ex/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "2f0c7bd2353d4a7fa4ff2d5c179262fa",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/4 [00:00<?, ?ex/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "{'commonsense_qa': {'validation': {'accuracy': {'': {\"_Answer: Let's think step by step._The answer is therefore\": 0.142857},\n",
       "    'gpt-3.5-turbo': {'None_kojima-01_kojima-A-E': 0.0}}}},\n",
       " 'med_qa': {'test': {'accuracy': {'': {\"_Answer: Let's think step by step._The answer is therefore\": 0.333333},\n",
       "    'gpt-3.5-turbo': {'None_kojima-01_kojima-A-E': 0.0}}}},\n",
       " 'open_book_qa': {'test': {'accuracy': {'': {\"_Answer: Let's think step by step._The answer is therefore\": 0.25},\n",
       "    'gpt-3.5-turbo': {'None_kojima-01_kojima-A-D': 0.0}}}}}"
      ]
     },
     "execution_count": 71,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#3\n",
    "multiple_ans_test.evaluate()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "either A, B, or C, depending on the situation.\n",
      "A and D. It is important to carefully read and analyze each answer choice before selecting the best answer.\n",
      "either A or C, depending on the situation and context. It is important to carefully consider all options and eliminate those that are clearly incorrect before making a final decision.\n",
      "A) adding salt or C) adding sand. It is important to understand the reasoning behind each option and how it can effectively reduce ice buildup on a sidewalk.\n",
      "D) Exhaustion.\n",
      "C) aggravation and E) stress.\n",
      "A) apartment and E) surface of earth. It is important to carefully read and consider all options before selecting the best answer.\n",
      "C) instruments, as it directly states that metal is used to make instruments. The other options may involve the use of metal, but they are not directly related to the question of what metal is used to make.\n",
      "E) hurt feelings and C) resentment.\n",
      "B) cardboard box or E) container.\n",
      "either C) forest or D) countryside.\n",
      "C) Increases levels of fetal hemoglobin (HgbF) and D) Decreases levels of HgbS. It is important to carefully read and understand the question and all answer choices before selecting the best answer.\n",
      "either B) Meningocele or D) Sensorineural hearing loss. It is important to carefully consider all the information provided in the question stem and eliminate answer choices that are not relevant or do not fit the clinical picture.\n",
      "either C) Nifedipine or D) Hydrochlorothiazide. It is important to carefully consider the patient's medical history and the potential side effects of each medication when selecting an appropriate treatment plan.\n"
     ]
    }
   ],
   "source": [
    "data = [('open_book_qa','test'),('commonsense_qa','validation'),('med_qa','test')]\n",
    "\n",
    "\n",
    "for pair in data:\n",
    "    for row in multiple_ans_test[pair[0]][pair[1]]:\n",
    "        text = row['generated_cot'][1]['answers'][0]['answer']\n",
    "        print(text)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "37df9bacd79448a79ecd5f4dfab58f98",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Creating json from Arrow format:   0%|          | 0/1 [00:00<?, ?ba/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "b05694eb04cc488eb39f06d46a391a75",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Creating json from Arrow format:   0%|          | 0/1 [00:00<?, ?ba/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "247eb5da0dcb4d6e8306f5cdeae7915f",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Creating json from Arrow format:   0%|          | 0/1 [00:00<?, ?ba/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "multiple_ans_test.dump(\"multiple_ans_test_1\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "OpenAI(cache=None, verbose=False, callback_manager=<langchain.callbacks.shared.SharedCallbackManager object at 0x7fcd11f25b50>, client=<class 'openai.api_resources.completion.Completion'>, model_name='text-davinci-003', temperature=0.0, max_tokens=256, top_p=1, frequency_penalty=0, presence_penalty=0, n=1, best_of=1, model_kwargs={}, openai_api_key=None, batch_size=20, request_timeout=None, logit_bias={}, max_retries=6, streaming=False)"
      ]
     },
     "execution_count": 44,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "llm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\"\"\"Test with prompt-2 for GPT-4 and Da Vinci\"\"\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\"\"\"Code notes:\n",
    "find right answer extraction from item\n",
    "\"\"\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Generating commonsense_qa...\n",
      "Generating med_qa...\n",
      "Generating open_book_qa...\n"
     ]
    }
   ],
   "source": [
    "input_dict = {\n",
    "    \"instruction\": \"\",\n",
    "    \"api_service\": \"chat_openai\",\n",
    "    \"model\": \"GPT-4\",\n",
    "    \"cot_trigger\": \"Answer: Let's think step by step.\",\n",
    "    \"answer_extraction\":  \"The answer is therefore\",\n",
    "    'reflect_answer_extraction': \"The above Answer to the Question contains multiple answers, however, only one answer is correct. Select one answer. The best answer is therefore\"\n",
    "}\n",
    "\n",
    "multiple_ans_test.extract_flexible(chain=single_reflect_answer_chain,input_dict=input_dict)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "35c2583512d2466aa0516a0df9c60122",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/7 [00:00<?, ?ex/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "a2856b169d624e2ba58fbbc1c0cba3f7",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/3 [00:00<?, ?ex/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "0454ff76d6364cb4abecc10806eb925b",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/4 [00:00<?, ?ex/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "{'commonsense_qa': {'validation': {'accuracy': {'GPT-4': {\"_Answer: Let's think step by step._The answer is therefore\": 0.428571},\n",
       "    'gpt-3.5-turbo': {'None_kojima-01_kojima-A-E': 0.0}}}},\n",
       " 'med_qa': {'test': {'accuracy': {'GPT-4': {\"_Answer: Let's think step by step._The answer is therefore\": 1.0},\n",
       "    'gpt-3.5-turbo': {'None_kojima-01_kojima-A-E': 0.0}}}},\n",
       " 'open_book_qa': {'test': {'accuracy': {'GPT-4': {\"_Answer: Let's think step by step._The answer is therefore\": 0.75},\n",
       "    'gpt-3.5-turbo': {'None_kojima-01_kojima-A-D': 0.0}}}}}"
      ]
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#1\n",
    "multiple_ans_test.evaluate()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "B) melting into sand\n",
      "A) It is made from rock\n",
      "C) making an adjustment to the rudder\n",
      "A) adding salt\n",
      "D) exhaustion\n",
      "E) stress.\n",
      "A) apartment\n",
      "C) instruments.\n",
      "C) resentment\n",
      "E) container.\n",
      "D) countryside\n",
      "C) Increases levels of fetal hemoglobin (HgbF).\n",
      "B) Meningocele\n",
      "C) Nifedipine.\n"
     ]
    }
   ],
   "source": [
    "data = [('open_book_qa','test'),('commonsense_qa','validation'),('med_qa','test')]\n",
    "\n",
    "\n",
    "for pair in data:\n",
    "    for row in multiple_ans_test[pair[0]][pair[1]]:\n",
    "        text = row['generated_cot'][1]['answers'][0]['answer']\n",
    "        print(text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "14161b2240a74862bda549a300c922a4",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Creating json from Arrow format:   0%|          | 0/1 [00:00<?, ?ba/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "cf0dddb509d44f36b5a45ddf43fa0a77",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Creating json from Arrow format:   0%|          | 0/1 [00:00<?, ?ba/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "30f4e3ae4030461bb6ad6fc1f267d3a1",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Creating json from Arrow format:   0%|          | 0/1 [00:00<?, ?ba/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "multiple_ans_test.dump(\"answer_correction_gpt_4\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Generating commonsense_qa...\n",
      "Generating med_qa...\n",
      "Generating open_book_qa...\n"
     ]
    }
   ],
   "source": [
    "input_dict = {\n",
    "    \"instruction\": \"\",\n",
    "    \"api_service\": \"openai\",\n",
    "    \"model\": \"text-davinci-003\",\n",
    "    \"cot_trigger\": \"Answer: Let's think step by step.\",\n",
    "    \"answer_extraction\":  \"The answer is therefore\",\n",
    "    'reflect_answer_extraction': \"The above Answer to the Question contains multiple answers, however, only one answer is correct. Select one answer. The best answer is therefore\"\n",
    "}\n",
    "\n",
    "multiple_ans_test.extract_flexible(chain=single_reflect_answer_chain,input_dict=input_dict)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "8d5efc1998754f2381f0039f776022e7",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/7 [00:00<?, ?ex/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "c46bf68388ca443a841e4679e1a17200",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/3 [00:00<?, ?ex/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "e1a9714fc1784d17945c622e2038d66e",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/4 [00:00<?, ?ex/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "{'commonsense_qa': {'validation': {'accuracy': {'gpt-3.5-turbo': {'None_kojima-01_kojima-A-E': 0.0},\n",
       "    'text-davinci-003': {\"_Answer: Let's think step by step._The answer is therefore\": 0.571429}}}},\n",
       " 'med_qa': {'test': {'accuracy': {'gpt-3.5-turbo': {'None_kojima-01_kojima-A-E': 0.0},\n",
       "    'text-davinci-003': {\"_Answer: Let's think step by step._The answer is therefore\": 1.0}}}},\n",
       " 'open_book_qa': {'test': {'accuracy': {'gpt-3.5-turbo': {'None_kojima-01_kojima-A-D': 0.0},\n",
       "    'text-davinci-003': {\"_Answer: Let's think step by step._The answer is therefore\": 0.75}}}}}"
      ]
     },
     "execution_count": 46,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "multiple_ans_test.evaluate()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "A) Mimicking other things\n",
      "\n",
      "A and D.\n",
      "\n",
      "Answer C: Making an adjustment to the rudder.\n",
      "\n",
      "A) adding salt\n",
      "\n",
      "Answer: D) Exhaustion\n",
      "\n",
      "C) aggravation.\n",
      "\n",
      "A) apartment.\n",
      "\n",
      "C) Instruments.\n",
      "\n",
      "C) Resentment.\n",
      "\n",
      "B) cardboard box or E) container.\n",
      "\n",
      "Answer: C) Forest\n",
      "\n",
      "C) Increases levels of fetal hemoglobin (HgbF)\n",
      "\n",
      "Answer: B) Meningocele\n",
      "\n",
      "Answer: C) Nifedipine\n"
     ]
    }
   ],
   "source": [
    "data = [('open_book_qa','test'),('commonsense_qa','validation'),('med_qa','test')]\n",
    "\n",
    "\n",
    "for pair in data:\n",
    "    for row in multiple_ans_test[pair[0]][pair[1]]:\n",
    "        text = row['generated_cot'][1]['answers'][0]['answer']\n",
    "        print(text)"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## One answer, extraction only"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/anaconda3/lib/python3.9/site-packages/scipy/__init__.py:146: UserWarning: A NumPy version >=1.16.5 and <1.23.0 is required for this version of SciPy (detected version 1.24.2\n",
      "  warnings.warn(f\"A NumPy version >={np_minversion} and <{np_maxversion}\"\n",
      "[nltk_data] Downloading package punkt to\n",
      "[nltk_data]     /Users/robertpraas/nltk_data...\n",
      "[nltk_data]   Package punkt is already up-to-date!\n"
     ]
    }
   ],
   "source": [
    "from cot import Collection\n",
    "multiple_ans_test = Collection.from_json(\"clean_chat_hard_multiple_ans.json\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Generating commonsense_qa...\n",
      "Generating med_qa...\n",
      "Generating open_book_qa...\n"
     ]
    }
   ],
   "source": [
    "#chatgpt\n",
    "\n",
    "input_dict = {\n",
    "    \"instruction\": \"\",\n",
    "    \"cot_trigger\": \"Answer: Let's think step by step.\",\n",
    "    \"answer_extraction\":  \"This is a multiple-choice question that only contains one correct answer, giving multiple answers is wrong. Only select the one best answer. The best answer is therefore\"\n",
    "}\n",
    "\n",
    "multiple_ans_test.extract_flexible(chain=answer_chain,input_dict=input_dict)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "8cfd24e6c3404141ab04bbea20f3be5c",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/7 [00:00<?, ?ex/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "533e1cec9c2240f88c1ffc6d0c3c1abc",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/3 [00:00<?, ?ex/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "bf965a814e7f4512a328274610a599ee",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/4 [00:00<?, ?ex/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "{'commonsense_qa': {'validation': {'accuracy': {'': {\"_Answer: Let's think step by step._This is a multiple-choice question that only contains one correct answer, giving multiple answers is wrong. Only select the one best answer. The best answer is therefore\": 0.571429},\n",
       "    'gpt-3.5-turbo': {'None_kojima-01_kojima-A-E': 0.0}}}},\n",
       " 'med_qa': {'test': {'accuracy': {'': {\"_Answer: Let's think step by step._This is a multiple-choice question that only contains one correct answer, giving multiple answers is wrong. Only select the one best answer. The best answer is therefore\": 0.666667},\n",
       "    'gpt-3.5-turbo': {'None_kojima-01_kojima-A-E': 0.0}}}},\n",
       " 'open_book_qa': {'test': {'accuracy': {'': {\"_Answer: Let's think step by step._This is a multiple-choice question that only contains one correct answer, giving multiple answers is wrong. Only select the one best answer. The best answer is therefore\": 0.25},\n",
       "    'gpt-3.5-turbo': {'None_kojima-01_kojima-A-D': 0.0}}}}}"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "multiple_ans_test.evaluate()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "either A, B, or C, depending on the situation.\n",
      "A) It is made from rock and D) It is made from metal.\n",
      "either A or C, depending on the situation and the sailor's preferences.\n",
      "A) adding salt.\n",
      "D) Exhaustion.\n",
      "C) aggravation.\n",
      "E) surface of earth.\n",
      "C) instruments, as it is the most specific and accurate answer to the question.\n",
      "C) resentment.\n",
      "either B) cardboard box or E) container.\n",
      "either C) forest or D) countryside.\n",
      "C) Increases levels of fetal hemoglobin (HgbF).\n",
      "B) Meningocele or D) Sensorineural hearing loss.\n",
      "either C) Nifedipine or D) Hydrochlorothiazide.\n"
     ]
    }
   ],
   "source": [
    "data = [('open_book_qa','test'),('commonsense_qa','validation'),('med_qa','test')]\n",
    "\n",
    "\n",
    "for pair in data:\n",
    "    for row in multiple_ans_test[pair[0]][pair[1]]:\n",
    "        text = row['generated_cot'][1]['answers'][0]['answer']\n",
    "        print(text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Generating commonsense_qa...\n",
      "Generating med_qa...\n",
      "Generating open_book_qa...\n"
     ]
    }
   ],
   "source": [
    "#GPT-4\n",
    "\n",
    "input_dict = {\n",
    "    \"instruction\": \"\",\n",
    "    \"cot_trigger\": \"Answer: Let's think step by step.\",\n",
    "    \"answer_extraction\":  \"This is a multiple-choice question that only contains one correct answer, giving multiple answers is wrong. Only select the one best answer. The best answer is therefore\"\n",
    "}\n",
    "\n",
    "multiple_ans_test.extract_flexible(chain=answer_chain,input_dict=input_dict)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "2ff32dc4e9a5492fa3ba6e19d987743d",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/7 [00:00<?, ?ex/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "32ee840d82504d28adb16a54ad11d36a",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/3 [00:00<?, ?ex/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "593b0dbf8312495d90c64af00199bf03",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/4 [00:00<?, ?ex/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "{'commonsense_qa': {'validation': {'accuracy': {'': {\"_Answer: Let's think step by step._This is a multiple-choice question that only contains one correct answer, giving multiple answers is wrong. Only select the one best answer. The best answer is therefore\": 0.5},\n",
       "    'gpt-3.5-turbo': {'None_kojima-01_kojima-A-E': 0.0}}}},\n",
       " 'med_qa': {'test': {'accuracy': {'': {\"_Answer: Let's think step by step._This is a multiple-choice question that only contains one correct answer, giving multiple answers is wrong. Only select the one best answer. The best answer is therefore\": 0.833333},\n",
       "    'gpt-3.5-turbo': {'None_kojima-01_kojima-A-E': 0.0}}}},\n",
       " 'open_book_qa': {'test': {'accuracy': {'': {\"_Answer: Let's think step by step._This is a multiple-choice question that only contains one correct answer, giving multiple answers is wrong. Only select the one best answer. The best answer is therefore\": 0.625},\n",
       "    'gpt-3.5-turbo': {'None_kojima-01_kojima-A-D': 0.0}}}}}"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "multiple_ans_test.evaluate()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "A) mimicking other things\n",
      "A) It is made from rock\n",
      "C) making an adjustment to the rudder\n",
      "A) adding salt\n",
      "D) exhaustion\n",
      "C) aggravation\n",
      "A) apartment\n",
      "C) instruments\n",
      "C) resentment\n",
      "B) cardboard box\n",
      "D) countryside\n",
      "C) Increases levels of fetal hemoglobin (HgbF).\n",
      "B) Meningocele, as it is a neural tube defect directly associated with carbamazepine use during pregnancy.\n",
      "C) Nifedipine, as it is a calcium channel blocker commonly used as a first-line anti-hypertensive medication and can cause lower extremity swelling as a side effect.\n"
     ]
    }
   ],
   "source": [
    "data = [('open_book_qa','test'),('commonsense_qa','validation'),('med_qa','test')]\n",
    "\n",
    "\n",
    "for pair in data:\n",
    "    for row in multiple_ans_test[pair[0]][pair[1]]:\n",
    "        text = row['generated_cot'][2]['answers'][0]['answer']\n",
    "        print(text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Generating commonsense_qa...\n",
      "Generating med_qa...\n",
      "Generating open_book_qa...\n"
     ]
    }
   ],
   "source": [
    "#Da Vinci\n",
    "\n",
    "input_dict = {\n",
    "    \"instruction\": \"\",\n",
    "    \"cot_trigger\": \"Answer: Let's think step by step.\",\n",
    "    \"answer_extraction\":  \"This is a multiple-choice question that only contains one correct answer, giving multiple answers is wrong. Only select the one best answer. The best answer is therefore\"\n",
    "}\n",
    "\n",
    "multiple_ans_test.extract_flexible(chain=answer_chain,input_dict=input_dict)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "73704fed750e49aa9de6e3db84e96ce6",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/7 [00:00<?, ?ex/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "d39130ab47064bdf93b1345e49066d7d",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/3 [00:00<?, ?ex/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "4bfe1b505bd9453d8539153fc0eb7a99",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/4 [00:00<?, ?ex/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "{'commonsense_qa': {'validation': {'accuracy': {'': {\"_Answer: Let's think step by step._This is a multiple-choice question that only contains one correct answer, giving multiple answers is wrong. Only select the one best answer. The best answer is therefore\": 0.52381},\n",
       "    'gpt-3.5-turbo': {'None_kojima-01_kojima-A-E': 0.0}}}},\n",
       " 'med_qa': {'test': {'accuracy': {'': {\"_Answer: Let's think step by step._This is a multiple-choice question that only contains one correct answer, giving multiple answers is wrong. Only select the one best answer. The best answer is therefore\": 0.777778},\n",
       "    'gpt-3.5-turbo': {'None_kojima-01_kojima-A-E': 0.0}}}},\n",
       " 'open_book_qa': {'test': {'accuracy': {'': {\"_Answer: Let's think step by step._This is a multiple-choice question that only contains one correct answer, giving multiple answers is wrong. Only select the one best answer. The best answer is therefore\": 0.75},\n",
       "    'gpt-3.5-turbo': {'None_kojima-01_kojima-A-D': 0.0}}}}}"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "multiple_ans_test.evaluate()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'id': '99bcd0b6-f9da-4c4d-9b46-2d1ada4f4ec1', 'fragments_version': '0.01', 'instruction': '', 'cot_trigger': \"Answer: Let's think step by step.\", 'cot_trigger_template': '', 'prompt_text': '', 'cot': '\\n\\nFirst, we need to understand that an octopus is a highly intelligent and adaptable creature. When faced with danger, it has several defense mechanisms to protect itself.\\n\\nOption A: Mimicking other things\\n\\nOctopuses are known for their ability to change color and texture to blend in with their surroundings. This is called camouflage, and it helps them avoid predators. However, it is not a foolproof defense mechanism, and sometimes an octopus may need to take more drastic measures.\\n\\nOption B: Melting into sand\\n\\nSome species of octopus have the ability to change not only their color and texture but also their shape. They can flatten themselves out and blend in with the sand or rocks on the ocean floor. This makes them almost invisible to predators.\\n\\nOption C: Creating new homes\\n\\nOctopuses are also known for their ability to build elaborate structures out of rocks and shells. These structures provide them with shelter and protection from predators. If an octopus is unable to swim to safety, it may try to build a new home to hide in.\\n\\nOption D: Mocking other fish\\n\\nThis option is not a valid defense mechanism for an octopus. While some species of octopus are known for their playful behavior and ability to mimic other creatures, this is not a defense mechanism. It is simply a way for the octopus to interact with its environment.\\n\\nTherefore, the correct answer is either A, B, or C, depending on the situation.', 'answers': [{'id': '348b886d-8e72-4f8e-abea-e905718a81c9', 'answer_extraction': 'This is a multiple-choice question that only contains one correct answer, giving multiple answers is wrong. Only select the one best answer. The best answer is therefore', 'answer_extraction_template': '', 'answer_extraction_text': '', 'answer': 'A) Mimicking other things', 'correct_answer': True}], 'author': '', 'date': '2023/03/30 10:28:05', 'api_service': '', 'model': \"{'name': '', 'temperature': 0, 'max_tokens': 800}\", 'comment': 'self_reflection cot', 'annotations': []}\n",
      "{'id': '7a994a01-a688-47ec-b8de-a8b95ca84512', 'fragments_version': '0.01', 'instruction': '', 'cot_trigger': \"Answer: Let's think step by step.\", 'cot_trigger_template': '', 'prompt_text': '', 'cot': \"\\n\\nA) It is made from rock - This is partially true, as the Earth's crust is made up of rocks. However, the Earth's interior is made up of molten rock and metal.\\n\\nB) It eats three meals a day - This is not true, as the Earth is not a living organism and does not require food.\\n\\nC) It has a loving family - This is not relevant to the Earth's sturdiness.\\n\\nD) It is made from metal - This is partially true, as the Earth's core is made up of mostly iron and nickel, which are metals. \\n\\nTherefore, the most accurate answer would be a combination of A and D - the Earth's sturdiness is due to its composition of rock and metal.\", 'answers': [{'id': 'df328ce8-97d2-4cc8-b114-6e50848a3410', 'answer_extraction': 'This is a multiple-choice question that only contains one correct answer, giving multiple answers is wrong. Only select the one best answer. The best answer is therefore', 'answer_extraction_template': '', 'answer_extraction_text': '', 'answer': 'A) It is made from rock', 'correct_answer': True}], 'author': '', 'date': '2023/03/30 10:28:06', 'api_service': '', 'model': \"{'name': '', 'temperature': 0, 'max_tokens': 800}\", 'comment': 'self_reflection cot', 'annotations': []}\n",
      "{'id': '5364fbb8-a208-4c9f-8bc6-3b21da04d87f', 'fragments_version': '0.01', 'instruction': '', 'cot_trigger': \"Answer: Let's think step by step.\", 'cot_trigger_template': '', 'prompt_text': '', 'cot': \"\\n\\nFirst, the sailor needs to set a course towards the shore. This can be done by adjusting the rudder, which controls the direction of the boat. So, option C is a possible answer.\\n\\nNext, the sailor needs to use the wind to move the boat towards the shore. This can be done by adjusting the sails to catch the wind. So, option A is also a possible answer.\\n\\nOption B, setting out to sea, is not a good choice if the sailor wants to reach the shore. It would take them further away from their destination.\\n\\nOption D, taking the afternoon off, is not a good choice either, as it would not help the sailor reach the shore.\\n\\nTherefore, the best answer is either A or C, depending on the situation and the sailor's preferences.\", 'answers': [{'id': '87ee41d4-f3d2-40dc-a309-d03885c315af', 'answer_extraction': 'This is a multiple-choice question that only contains one correct answer, giving multiple answers is wrong. Only select the one best answer. The best answer is therefore', 'answer_extraction_template': '', 'answer_extraction_text': '', 'answer': 'C) Making an adjustment to the rudder.', 'correct_answer': True}], 'author': '', 'date': '2023/03/30 10:28:07', 'api_service': '', 'model': \"{'name': '', 'temperature': 0, 'max_tokens': 800}\", 'comment': 'self_reflection cot', 'annotations': []}\n",
      "{'id': '173d3c67-0be7-4ac9-845c-74e0d607be86', 'fragments_version': '0.01', 'instruction': '', 'cot_trigger': \"Answer: Let's think step by step.\", 'cot_trigger_template': '', 'prompt_text': '', 'cot': '\\n\\nFirst, we need to understand why ice forms on a sidewalk. Ice forms when the temperature drops below freezing, causing any moisture on the sidewalk to freeze. This can create a dangerous situation for pedestrians, as the ice can be slippery and cause falls.\\n\\nTo reduce the ice buildup on a sidewalk, we need to find a way to melt the ice or create traction on the surface. Adding water would not be effective, as it would simply freeze again and create more ice. Litter would not be helpful either, as it could create a hazard for pedestrians and would not provide any traction.\\n\\nThe two most effective options are adding salt or adding sand. Salt lowers the freezing point of water, causing the ice to melt. Sand provides traction on the surface, making it less slippery. Both options can be effective, depending on the situation.\\n\\nTherefore, the correct answer is either A) adding salt or C) adding sand.', 'answers': [{'id': '058b93c1-e332-41dd-b78a-9d9e624694ac', 'answer_extraction': 'This is a multiple-choice question that only contains one correct answer, giving multiple answers is wrong. Only select the one best answer. The best answer is therefore', 'answer_extraction_template': '', 'answer_extraction_text': '', 'answer': 'A) Adding salt.', 'correct_answer': True}], 'author': '', 'date': '2023/03/30 10:28:08', 'api_service': '', 'model': \"{'name': '', 'temperature': 0, 'max_tokens': 800}\", 'comment': 'self_reflection cot', 'annotations': []}\n",
      "{'id': '80381515-2545-49ff-a2d7-8ea2e1b45c42', 'fragments_version': '0.01', 'instruction': '', 'cot_trigger': \"Answer: Let's think step by step.\", 'cot_trigger_template': '', 'prompt_text': '', 'cot': '\\n\\nFirst, we need to understand that running requires physical exertion and energy. Therefore, the answer options that relate to physical limitations are more likely to be correct.\\n\\nA) Mushroom: This option is unlikely to be correct as a mushroom would not suddenly appear and stop someone from running.\\n\\nB) Falling down: This option is possible as a sudden obstacle or uneven surface could cause someone to trip and fall, stopping them from running.\\n\\nC) Sweating: This option is unlikely to be correct as sweating is a natural bodily response to physical activity and would not necessarily stop someone from running.\\n\\nD) Exhaustion: This option is highly likely to be correct as exhaustion is a physical limitation that can suddenly stop someone from running.\\n\\nE) Getting tired: This option is similar to option D and is also likely to be correct as getting tired is a physical limitation that can suddenly stop someone from running.\\n\\nTherefore, the correct answers are B, D, and E.', 'answers': [{'id': 'f4e397a8-66a6-4786-9836-425f22cb51f2', 'answer_extraction': 'This is a multiple-choice question that only contains one correct answer, giving multiple answers is wrong. Only select the one best answer. The best answer is therefore', 'answer_extraction_template': '', 'answer_extraction_text': '', 'answer': 'D) Exhaustion.', 'correct_answer': False}], 'author': '', 'date': '2023/03/30 10:27:56', 'api_service': '', 'model': \"{'name': '', 'temperature': 0, 'max_tokens': 800}\", 'comment': 'self_reflection cot', 'annotations': []}\n",
      "{'id': '71972f98-44e2-430f-a827-bc50aff22f79', 'fragments_version': '0.01', 'instruction': '', 'cot_trigger': \"Answer: Let's think step by step.\", 'cot_trigger_template': '', 'prompt_text': '', 'cot': '\\n\\nRunning errands with kids can be challenging, especially if they are screaming. This can lead to:\\n\\nC) aggravation: The constant noise and disruption can be frustrating and irritating.\\n\\nE) stress: The added pressure of trying to complete tasks while managing screaming kids can be stressful.\\n\\nB) insanity: The chaos and noise can be overwhelming and make one feel like they are losing their mind.\\n\\nD) tiredness: The constant stimulation and effort required to manage the situation can be exhausting.\\n\\nA) efficiency: Unfortunately, running errands with screaming kids is unlikely to increase efficiency. It may actually slow down the process as one tries to manage the situation.\\n\\nTherefore, the correct answer is C) aggravation, E) stress, B) insanity, and D) tiredness.', 'answers': [{'id': '2f5b6d19-e890-466a-ae30-e925aee4ea8e', 'answer_extraction': 'This is a multiple-choice question that only contains one correct answer, giving multiple answers is wrong. Only select the one best answer. The best answer is therefore', 'answer_extraction_template': '', 'answer_extraction_text': '', 'answer': 'C) Aggravation.', 'correct_answer': True}], 'author': '', 'date': '2023/03/30 10:27:57', 'api_service': '', 'model': \"{'name': '', 'temperature': 0, 'max_tokens': 800}\", 'comment': 'self_reflection cot', 'annotations': []}\n",
      "{'id': 'b0cb1d2b-213a-43ab-a52b-ef676e4ff115', 'fragments_version': '0.01', 'instruction': '', 'cot_trigger': \"Answer: Let's think step by step.\", 'cot_trigger_template': '', 'prompt_text': '', 'cot': '\\n\\nFirst, we can eliminate options B, C, and D as they are not places where people typically live. \\n\\nOption A, an apartment, is a common living arrangement for many people in urban areas. \\n\\nOption E, the surface of the earth, is where the majority of people live. However, some people may live in other environments such as on boats, in underground homes, or in treehouses. \\n\\nTherefore, the answer is A) apartment and E) surface of earth.', 'answers': [{'id': '6b5f60ec-9216-4191-9a32-73df1bc93a45', 'answer_extraction': 'This is a multiple-choice question that only contains one correct answer, giving multiple answers is wrong. Only select the one best answer. The best answer is therefore', 'answer_extraction_template': '', 'answer_extraction_text': '', 'answer': 'A) apartment.', 'correct_answer': False}], 'author': '', 'date': '2023/03/30 10:27:58', 'api_service': '', 'model': \"{'name': '', 'temperature': 0, 'max_tokens': 800}\", 'comment': 'self_reflection cot', 'annotations': []}\n",
      "{'id': 'fe368ce1-a934-40cb-8764-82f01c5137ea', 'fragments_version': '0.01', 'instruction': '', 'cot_trigger': \"Answer: Let's think step by step.\", 'cot_trigger_template': '', 'prompt_text': '', 'cot': \"\\n\\nFirst, we need to understand what metal is. Metal is a solid material that is typically hard, shiny, malleable, ductile, and a good conductor of heat and electricity. \\n\\nNow, let's look at the options:\\n\\nA) Junkyard - While metal scraps may end up in a junkyard, metal itself is not used to make a junkyard.\\n\\nB) Ore - Ore is a mineral that contains metal, but it is not the metal itself.\\n\\nC) Instruments - Yes, metal is commonly used to make instruments such as guitars, drums, and trumpets.\\n\\nD) Metal fabrication shop - A metal fabrication shop is a place where metal is processed and shaped into various products, so metal is definitely used here.\\n\\nE) Bowls - Yes, metal is commonly used to make bowls, along with other kitchenware such as pots and pans.\\n\\nTherefore, the correct answer is C) instruments, D) metal fabrication shop, and E) bowls.\", 'answers': [{'id': '0ba5c1fb-e984-469e-8633-6e8ec3f6ff10', 'answer_extraction': 'This is a multiple-choice question that only contains one correct answer, giving multiple answers is wrong. Only select the one best answer. The best answer is therefore', 'answer_extraction_template': '', 'answer_extraction_text': '', 'answer': 'C) Instruments.', 'correct_answer': True}], 'author': '', 'date': '2023/03/30 10:27:59', 'api_service': '', 'model': \"{'name': '', 'temperature': 0, 'max_tokens': 800}\", 'comment': 'self_reflection cot', 'annotations': []}\n",
      "{'id': 'b2c50dfc-99b1-49de-9482-76c4104b9288', 'fragments_version': '0.01', 'instruction': '', 'cot_trigger': \"Answer: Let's think step by step.\", 'cot_trigger_template': '', 'prompt_text': '', 'cot': '\\n\\nWhen someone judges a person guilty, they often feel:\\n\\nC) Resentment - because they feel unfairly judged and misunderstood.\\n\\nE) Hurt feelings - because they feel attacked and criticized.\\n\\nD) Judge feelings - because they may feel judged and evaluated by others.\\n\\nA) Controversy - because the judgment may be disputed or challenged.\\n\\nB) Responsibility - because they may feel accountable for their actions or behavior.\\n\\nTherefore, the answer is a combination of C) resentment and E) hurt feelings.', 'answers': [{'id': '1666e976-a7b2-4f46-93ce-a781efff30de', 'answer_extraction': 'This is a multiple-choice question that only contains one correct answer, giving multiple answers is wrong. Only select the one best answer. The best answer is therefore', 'answer_extraction_template': '', 'answer_extraction_text': '', 'answer': 'C) Resentment.', 'correct_answer': True}], 'author': '', 'date': '2023/03/30 10:28:00', 'api_service': '', 'model': \"{'name': '', 'temperature': 0, 'max_tokens': 800}\", 'comment': 'self_reflection cot', 'annotations': []}\n",
      "{'id': '01b5d7eb-0893-4847-bd07-1425b38997c6', 'fragments_version': '0.01', 'instruction': '', 'cot_trigger': \"Answer: Let's think step by step.\", 'cot_trigger_template': '', 'prompt_text': '', 'cot': '\\n\\nFirst, we need to understand the question. The question is asking what you can buy at a shop to put your spare unused things. \\n\\nOption A, basement, is not something you can buy at a shop. It is a part of a house or building. \\n\\nOption B, cardboard box, is a possible answer. You can buy cardboard boxes at a shop to store your spare unused things. \\n\\nOption C, ocean floor, is not something you can buy at a shop. It is a natural environment. \\n\\nOption D, high school, is not something you can buy at a shop. It is an educational institution. \\n\\nOption E, container, is another possible answer. You can buy containers at a shop to store your spare unused things. \\n\\nTherefore, the correct answer is either B) cardboard box or E) container, depending on personal preference and the type of items being stored.', 'answers': [{'id': 'b9724113-a5c3-4fbc-82d0-726188b3e659', 'answer_extraction': 'This is a multiple-choice question that only contains one correct answer, giving multiple answers is wrong. Only select the one best answer. The best answer is therefore', 'answer_extraction_template': '', 'answer_extraction_text': '', 'answer': 'B) cardboard box.', 'correct_answer': False}], 'author': '', 'date': '2023/03/30 10:28:01', 'api_service': '', 'model': \"{'name': '', 'temperature': 0, 'max_tokens': 800}\", 'comment': 'self_reflection cot', 'annotations': []}\n",
      "{'id': 'f1fbabb2-4f75-416d-94ce-bcb269c05a70', 'fragments_version': '0.01', 'instruction': '', 'cot_trigger': \"Answer: Let's think step by step.\", 'cot_trigger_template': '', 'prompt_text': '', 'cot': '\\n\\nFirst, we know that David is on vacation. This rules out options A and E, as they suggest he is in the sky or on a roof, which is unlikely to be a vacation destination. \\n\\nNext, we know that David is observing nesting birds, which suggests he is in a natural setting. This rules out option B, as it is too broad and could include many different types of vacation destinations. \\n\\nFinally, the most likely option is C, forest, or D, countryside, as these are both natural settings where birds are likely to nest. Without more information, it is impossible to determine which of these options is correct, but either one would be a reasonable guess based on the information given. \\n\\nTherefore, the answer is either C) forest or D) countryside.', 'answers': [{'id': 'e6f96260-e0e8-4c4e-b19c-3fc2d06922ed', 'answer_extraction': 'This is a multiple-choice question that only contains one correct answer, giving multiple answers is wrong. Only select the one best answer. The best answer is therefore', 'answer_extraction_template': '', 'answer_extraction_text': '', 'answer': 'C) forest.', 'correct_answer': True}], 'author': '', 'date': '2023/03/30 10:28:01', 'api_service': '', 'model': \"{'name': '', 'temperature': 0, 'max_tokens': 800}\", 'comment': 'self_reflection cot', 'annotations': []}\n",
      "{'id': '3f6a28e7-c307-42d0-ba82-a51fa1215f27', 'fragments_version': '0.01', 'instruction': '', 'cot_trigger': \"Answer: Let's think step by step.\", 'cot_trigger_template': '', 'prompt_text': '', 'cot': \"\\n\\nPatients with sickle cell anemia have a mutation that results in the production of HgbS, which causes their red blood cells to sickle. Hydroxyurea is a medication commonly used to treat sickle cell anemia. So, what does hydroxyurea do to the hemoglobin physiology of these patients?\\n\\nA) Increases oxygen carrying capacity of hemoglobin - This is not correct. Hydroxyurea does not directly increase the oxygen carrying capacity of hemoglobin.\\n\\nB) Decreases oxygen carrying capacity of hemoglobin - This is not correct. Hydroxyurea does not directly decrease the oxygen carrying capacity of hemoglobin.\\n\\nC) Increases levels of fetal hemoglobin (HgbF) - This is correct. Hydroxyurea can increase the levels of fetal hemoglobin (HgbF) in patients with sickle cell anemia. HgbF is a type of hemoglobin that is normally present in fetuses and newborns, but it decreases as we age. HgbF does not sickle, so increasing its levels can help prevent sickling of red blood cells in patients with sickle cell anemia.\\n\\nD) Decreases levels of HgbS - This is correct. Hydroxyurea can decrease the levels of HgbS in patients with sickle cell anemia. This is because hydroxyurea can stimulate the production of HgbF, which can replace some of the HgbS in the patient's red blood cells.\\n\\nE) Decreases levels of fetal hemoglobin (HgbF) - This is not correct. Hydroxyurea can actually increase the levels of HgbF in patients with sickle cell anemia, as mentioned in option C.\\n\\nTherefore, the correct answer is C) Increases levels of fetal hemoglobin (HgbF) and D) Decreases levels of HgbS.\", 'answers': [{'id': 'ad1bf6c7-8170-43a5-9c55-23b987059f5c', 'answer_extraction': 'This is a multiple-choice question that only contains one correct answer, giving multiple answers is wrong. Only select the one best answer. The best answer is therefore', 'answer_extraction_template': '', 'answer_extraction_text': '', 'answer': 'C) Increases levels of fetal hemoglobin (HgbF)', 'correct_answer': True}], 'author': '', 'date': '2023/03/30 10:28:03', 'api_service': '', 'model': \"{'name': '', 'temperature': 0, 'max_tokens': 800}\", 'comment': 'self_reflection cot', 'annotations': []}\n",
      "{'id': 'bb9a8aa2-3087-4595-9358-96fcc066a139', 'fragments_version': '0.01', 'instruction': '', 'cot_trigger': \"Answer: Let's think step by step.\", 'cot_trigger_template': '', 'prompt_text': '', 'cot': '\\n\\nThe patient is pregnant and has been taking carbamazepine for her seizure disorder. Carbamazepine is known to increase the risk of certain complications in pregnancy, including neural tube defects and developmental delays. Therefore, options B and D, which are both related to neural tube defects, are potential complications.\\n\\nOption A, renal dysplasia, is not typically associated with carbamazepine use in pregnancy.\\n\\nOption C, teeth discoloration, is a potential side effect of tetracycline use during pregnancy, but is not related to carbamazepine use.\\n\\nOption E, vaginal clear cell carcinoma, is a rare complication associated with exposure to diethylstilbestrol (DES) in utero, but is not related to carbamazepine use.\\n\\nTherefore, the correct answer is either B) Meningocele or D) Sensorineural hearing loss.', 'answers': [{'id': '07578a7a-2d45-428d-ad03-c38e00c3c52e', 'answer_extraction': 'This is a multiple-choice question that only contains one correct answer, giving multiple answers is wrong. Only select the one best answer. The best answer is therefore', 'answer_extraction_template': '', 'answer_extraction_text': '', 'answer': '\\nD) Sensorineural hearing loss', 'correct_answer': False}], 'author': '', 'date': '2023/03/30 10:28:03', 'api_service': '', 'model': \"{'name': '', 'temperature': 0, 'max_tokens': 800}\", 'comment': 'self_reflection cot', 'annotations': []}\n",
      "{'id': '3399ec8b-fb5a-437f-a53d-0eaaf694c832', 'fragments_version': '0.01', 'instruction': '', 'cot_trigger': \"Answer: Let's think step by step.\", 'cot_trigger_template': '', 'prompt_text': '', 'cot': \"\\n\\nThe patient has a history of poorly controlled hypertension and is taking enalapril, which is an ACE inhibitor. The physician adds another anti-hypertensive medication, which suggests that the patient's blood pressure was not well controlled with enalapril alone. \\n\\nThe patient returns with new onset lower extremity swelling, which is a common side effect of some anti-hypertensive medications. \\n\\nMetoprolol is a beta-blocker and is not typically associated with lower extremity swelling. \\n\\nVerapamil is a calcium channel blocker and can cause lower extremity swelling, but it is not commonly used as a first-line anti-hypertensive medication. \\n\\nNifedipine is also a calcium channel blocker and can cause lower extremity swelling. It is commonly used as a first-line anti-hypertensive medication. \\n\\nHydrochlorothiazide is a diuretic and can cause lower extremity swelling as a side effect. It is commonly used as a first-line anti-hypertensive medication. \\n\\nSpironolactone is a potassium-sparing diuretic and can also cause lower extremity swelling. It is not commonly used as a first-line anti-hypertensive medication. \\n\\nBased on the information provided, the most likely medication that was prescribed to this patient is either nifedipine or hydrochlorothiazide. The correct answer is therefore either C) Nifedipine or D) Hydrochlorothiazide.\", 'answers': [{'id': 'ccf6f78c-fec8-4a87-8d90-b4896ef6fe70', 'answer_extraction': 'This is a multiple-choice question that only contains one correct answer, giving multiple answers is wrong. Only select the one best answer. The best answer is therefore', 'answer_extraction_template': '', 'answer_extraction_text': '', 'answer': 'C) Nifedipine', 'correct_answer': True}], 'author': '', 'date': '2023/03/30 10:28:04', 'api_service': '', 'model': \"{'name': '', 'temperature': 0, 'max_tokens': 800}\", 'comment': 'self_reflection cot', 'annotations': []}\n"
     ]
    }
   ],
   "source": [
    "data = [('open_book_qa','test'),('commonsense_qa','validation'),('med_qa','test')]\n",
    "\n",
    "\n",
    "for pair in data:\n",
    "    for row in multiple_ans_test[pair[0]][pair[1]]:\n",
    "        text = row['generated_cot'][3]\n",
    "        print(text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "6057d33e7e034fb89ae39dac1129feee",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Creating json from Arrow format:   0%|          | 0/1 [00:00<?, ?ba/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "9acb54ca91a543ea94ae3fe2554b0511",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Creating json from Arrow format:   0%|          | 0/1 [00:00<?, ?ba/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "3035005857184a22bf1e5a4d1c5718ff",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Creating json from Arrow format:   0%|          | 0/1 [00:00<?, ?ba/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "multiple_ans_test.dump(\"Force_one_ans_three_models\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Single reflection prompt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "f48d40a7336d4443a116a06db3971039",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/100 [00:00<?, ?ex/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "from cot import Collection\n",
    "coll = Collection.load_thoughtsource_100(names=['worldtree'],load_pregenerated_cots=False) #random_sample=False?\n",
    "coll = coll.select(split=\"all\", number_samples=10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "| Name      | Train   | Valid   |   Test |\n",
       "|-----------|---------|---------|--------|\n",
       "| worldtree | -       | -       |     10 |\n",
       "\n",
       "Not loaded: ['aqua', 'asdiv', 'commonsense_qa', 'entailment_bank', 'gsm8k', 'mawps', 'med_qa', 'medmc_qa', 'open_book_qa', 'pubmed_qa', 'qed', 'strategy_qa', 'svamp']"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "coll"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# #check answer extraction\n",
    "# input_dict = {\n",
    "#     \"cot_trigger\": \"Answer: Let's think step by step.\",\n",
    "#     \"answer_extraction\": \"Therefore, the answer is\", \n",
    "#     'answer':\"\", \n",
    "#     'cot': \"\", \n",
    "#     'reflection_prompt':\"The goal is to correct the Answer (generated by a language model) if needed, let's think step by step\" ,\n",
    "#     'reflect_answer_extraction':'Based on the reflection, the answer is therefore',\n",
    "#     'model_name':\"gpt-4\",\n",
    "#     'temperature': 0,\n",
    "#     'max_tokens':800\n",
    "# }\n",
    "# coll.metareason_flexible(chain=reflect_overall_chain,input_dict=input_dict)\n",
    "# coll.evaluate()\n",
    "# #coll.dump(\"single_meta.json\")\n",
    "# coll"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "config={\n",
    "    # We compare three different prompts for the chain of thought generation:\n",
    "    # \"Answer: Let's think step by step.\" and 'Answer: We should think about this step by step.', and \"Answer: First,\" \n",
    "    \"cot_trigger_keys\": ['praas-01'],\n",
    "\n",
    "    # We use the same answer extraction prompt for all three prompts\n",
    "    # \"Therefore, among A through D, the answer is\"\n",
    "    \"answer_extraction_keys\": ['auto-kojima'],#['kojima-A-D'], \n",
    "    \n",
    "    \"author\" : \"Robert\",\n",
    "    \"api_service\": \"openai_chat\", \n",
    "    \"engine\": \"gpt-4\", \n",
    "    \"temperature\": 0,\n",
    "    \"max_tokens\": 512,\n",
    "    \"verbose\": False,\n",
    "    \"warn\": True,\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Generating worldtree...\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "829832e761b248438baf3b30a8abe154",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/5 [00:00<?, ?ex/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "coll.generate(config=config)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "42615c9ecaf14a9889a8f10f83021a67",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Creating json from Arrow format:   0%|          | 0/1 [00:00<?, ?ba/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "coll.dump(\"single_meta_experiment.json\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'id': '52d6be0f-3757-46a5-961f-ec1e85730ac2',\n",
       " 'fragments_version': '0.01',\n",
       " 'instruction': None,\n",
       " 'cot_trigger': 'praas-01',\n",
       " 'cot_trigger_template': '{instruction}\\n\\n{question}\\n{answer_choices}\\n\\n{cot_trigger}',\n",
       " 'prompt_text': '',\n",
       " 'cot': \"Answer: A) the mass of Earth.\\n\\nProblems with the answer: The answer is correct, but it does not provide a complete explanation of how gravity works. It only mentions the mass of Earth as the cause of gravity but does not mention the role of distance between objects and the gravitational constant.\\n\\nImproved answer: Gravity on Earth is caused by the mass of Earth, the distance between objects, and the gravitational constant. The force of gravity is directly proportional to the product of the masses of the objects and inversely proportional to the square of the distance between their centers. This relationship is described by Newton's law of universal gravitation.\",\n",
       " 'answers': [{'id': 'ead68434-f09f-49b1-9c6b-16ba80913022',\n",
       "   'answer_extraction': 'kojima-A-D',\n",
       "   'answer_extraction_template': '{instruction}\\n\\n{question}\\n{answer_choices}\\n\\n{cot_trigger}{cot}\\n{answer_extraction}',\n",
       "   'answer_extraction_text': '',\n",
       "   'answer': \"A) the mass of Earth, but it is also influenced by the distance between objects and the gravitational constant, as described by Newton's law of universal gravitation.\",\n",
       "   'correct_answer': None}],\n",
       " 'author': 'Robert',\n",
       " 'date': '2023/04/14 17:24:33',\n",
       " 'api_service': 'openai_chat',\n",
       " 'model': \"{'name': 'gpt-4', 'temperature': 0, 'max_tokens': 512}\",\n",
       " 'comment': '',\n",
       " 'annotations': []}"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "coll['worldtree']['test'][0]['generated_cot'][27]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "# On GPT-4 hard dataset\n",
    "\n",
    "gpt_4_hard = Collection.from_json(\"filtered_thoughtsource_100_gpt-4_false_only_gpt-4_cots.json\")\n",
    "gpt_4_hard.select_generated_cots(cot_trigger = \"zhou-01\", api_service='openai_chat',model = 'gpt-4',answer=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "| Name           | Train   | Valid   | Test   |\n",
       "|----------------|---------|---------|--------|\n",
       "| commonsense_qa | -       | 28      | -      |\n",
       "| med_qa         | -       | -       | 24     |\n",
       "| medmc_qa       | -       | 30      | -      |\n",
       "| open_book_qa   | -       | -       | 5      |\n",
       "| strategy_qa    | 20      | -       | -      |\n",
       "| worldtree      | -       | -       | 1      |\n",
       "\n",
       "Not loaded: ['aqua', 'asdiv', 'entailment_bank', 'gsm8k', 'mawps', 'pubmed_qa', 'qed', 'svamp']"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gpt_4_hard = gpt_4_hard.filter(lambda x: len(x[\"generated_cot\"])==1)\n",
    "gpt_4_hard"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "gpt_4_hard_backup = gpt_4_hard"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "| Name         | Train   | Valid   |   Test |\n",
       "|--------------|---------|---------|--------|\n",
       "| open_book_qa | -       | -       |      5 |\n",
       "| worldtree    | -       | -       |      1 |\n",
       "\n",
       "Not loaded: ['aqua', 'asdiv', 'commonsense_qa', 'entailment_bank', 'gsm8k', 'mawps', 'med_qa', 'medmc_qa', 'pubmed_qa', 'qed', 'strategy_qa', 'svamp']"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gpt_4_hard.unload_datasets([\"med_qa\",\"medmc_qa\",\"commonsense_qa\",\"strategy_qa\"])\n",
    "gpt_4_hard"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Generating open_book_qa...\n",
      "Generating worldtree...\n"
     ]
    }
   ],
   "source": [
    "gpt_4_hard.generate(config=config)\n",
    "gpt_4_hard.evaluate()\n",
    "gpt_4_hard.dump(\"single_meta_experiment_2.json\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Second prompt\n",
    "config={\n",
    "    # We compare three different prompts for the chain of thought generation:\n",
    "    # \"Answer: Let's think step by step.\" and 'Answer: We should think about this step by step.', and \"Answer: First,\" \n",
    "    \"cot_trigger_keys\": ['praas-02'],\n",
    "\n",
    "    # We use the same answer extraction prompt for all three prompts\n",
    "    # \"Therefore, among A through D, the answer is\"\n",
    "    \"answer_extraction_keys\": ['auto-kojima'],#['kojima-A-D'], \n",
    "    \n",
    "    \"author\" : \"Robert\",\n",
    "    \"api_service\": \"openai_chat\", \n",
    "    \"engine\": \"gpt-4\", \n",
    "    \"temperature\": 0,\n",
    "    \"max_tokens\": 512,\n",
    "    \"verbose\": False,\n",
    "    \"warn\": True,\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "from cot import Collection\n",
    "gpt_4_hard = Collection.from_json(\"filtered_thoughtsource_100_gpt-4_false_only_gpt-4_cots.json\")\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "| Name           | Train   | Valid   | Test   |\n",
       "|----------------|---------|---------|--------|\n",
       "| commonsense_qa | -       | 28      | -      |\n",
       "| med_qa         | -       | -       | 24     |\n",
       "| medmc_qa       | -       | 30      | -      |\n",
       "| open_book_qa   | -       | -       | 5      |\n",
       "| strategy_qa    | 20      | -       | -      |\n",
       "| worldtree      | -       | -       | 1      |\n",
       "\n",
       "Not loaded: ['aqua', 'asdiv', 'entailment_bank', 'gsm8k', 'mawps', 'pubmed_qa', 'qed', 'svamp']"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gpt_4_hard.select_generated_cots(cot_trigger = \"zhou-01\", api_service='openai_chat',model = 'gpt-4',answer=False)\n",
    "gpt_4_hard = gpt_4_hard.filter(lambda x: len(x[\"generated_cot\"])==1)\n",
    "gpt_4_hard"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [],
   "source": [
    "gpt_4_hard.unload_datasets([\"commonsense_qa\",\"medmc_qa\",\"worldtree\",\"med_qa\",'open_book_qa'])\n",
    "# gpt_4_hard.select_generated_cots(cot_trigger = \"zhou-01\", api_service='openai_chat',model = 'gpt-4',answer=False)\n",
    "# gpt_4_hard"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "| Name        |   Train | Valid   | Test   |\n",
       "|-------------|---------|---------|--------|\n",
       "| strategy_qa |      20 | -       | -      |\n",
       "\n",
       "Not loaded: ['aqua', 'asdiv', 'commonsense_qa', 'entailment_bank', 'gsm8k', 'mawps', 'med_qa', 'medmc_qa', 'open_book_qa', 'pubmed_qa', 'qed', 'svamp', 'worldtree']"
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gpt_4_hard"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Generating open_book_qa...\n",
      "Generating worldtree...\n"
     ]
    }
   ],
   "source": [
    "gpt_4_hard.generate(config=config)\n",
    "gpt_4_hard.evaluate()\n",
    "gpt_4_hard.dump(\"single_meta_experiment_prompt_2.json\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Third prompt\n",
    "config={\n",
    "    # We compare three different prompts for the chain of thought generation:\n",
    "    # \"Answer: Let's think step by step.\" and 'Answer: We should think about this step by step.', and \"Answer: First,\" \n",
    "    \"cot_trigger_keys\": ['praas-03'],\n",
    "\n",
    "    # We use the same answer extraction prompt for all three prompts\n",
    "    # \"Therefore, among A through D, the answer is\"\n",
    "    \"answer_extraction_keys\": ['auto-kojima'], #['kojima-A-D'], \n",
    "    \n",
    "    \"author\" : \"Robert\",\n",
    "    \"api_service\": \"openai_chat\", \n",
    "    \"engine\": \"gpt-4\", \n",
    "    \"temperature\": 0,\n",
    "    \"max_tokens\": 512,\n",
    "    \"verbose\": False,\n",
    "    \"warn\": True,\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "| Name         | Train   | Valid   |   Test |\n",
       "|--------------|---------|---------|--------|\n",
       "| open_book_qa | -       | -       |      5 |\n",
       "| worldtree    | -       | -       |      1 |\n",
       "\n",
       "Not loaded: ['aqua', 'asdiv', 'commonsense_qa', 'entailment_bank', 'gsm8k', 'mawps', 'med_qa', 'medmc_qa', 'pubmed_qa', 'qed', 'strategy_qa', 'svamp']"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gpt_4_hard"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Generating open_book_qa...\n",
      "Generating worldtree...\n"
     ]
    }
   ],
   "source": [
    "gpt_4_hard.generate(config=config)\n",
    "gpt_4_hard.evaluate()\n",
    "gpt_4_hard.dump(\"single_meta_experiment_prompt_3.json\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Generating worldtree...\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "dd0ba07e8cdc44d5b1b276b5b9becf84",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/10 [00:00<?, ?ex/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "ccd7b91880b64d278c5af34870dcbc6a",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/10 [00:00<?, ?ex/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "4f46803567da45bc9b769fe01ae5ebba",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Creating json from Arrow format:   0%|          | 0/1 [00:00<?, ?ba/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "coll.generate(config=config)\n",
    "coll.evaluate()\n",
    "coll.dump(\"single_meta_experiment_prompt_2_worldtree.json\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Generating commonsense_qa...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Retrying langchain.chat_models.openai.ChatOpenAI.completion_with_retry.<locals>._completion_with_retry in 4.0 seconds as it raised APIError: Bad gateway. {\"error\":{\"code\":502,\"message\":\"Bad gateway.\",\"param\":null,\"type\":\"cf_bad_gateway\"}} 502 {'error': {'code': 502, 'message': 'Bad gateway.', 'param': None, 'type': 'cf_bad_gateway'}} {'Date': 'Mon, 17 Apr 2023 08:24:51 GMT', 'Content-Type': 'application/json', 'Content-Length': '84', 'Connection': 'keep-alive', 'X-Frame-Options': 'SAMEORIGIN', 'Referrer-Policy': 'same-origin', 'Cache-Control': 'private, max-age=0, no-store, no-cache, must-revalidate, post-check=0, pre-check=0', 'Expires': 'Thu, 01 Jan 1970 00:00:01 GMT', 'Server': 'cloudflare', 'CF-RAY': '7b933e4e6b6577eb-VIE'}.\n"
     ]
    }
   ],
   "source": [
    "#try commmonsense prompt 2\n",
    "\n",
    "gpt_4_hard.generate(config=config)\n",
    "gpt_4_hard.evaluate()\n",
    "gpt_4_hard.dump(\"prompt_2_cs.json\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "config={\n",
    "    # We compare three different prompts for the chain of thought generation:\n",
    "    # \"Answer: Let's think step by step.\" and 'Answer: We should think about this step by step.', and \"Answer: First,\" \n",
    "    \"cot_trigger_keys\": ['praas-03'],\n",
    "\n",
    "    # We use the same answer extraction prompt for all three prompts\n",
    "    # \"Therefore, among A through D, the answer is\"\n",
    "    \"answer_extraction_keys\": ['auto-kojima'],#['kojima-A-D'], \n",
    "    \n",
    "    \"author\" : \"Robert\",\n",
    "    \"api_service\": \"openai_chat\", \n",
    "    \"engine\": \"gpt-4\", \n",
    "    \"temperature\": 0,\n",
    "    \"max_tokens\": 512,\n",
    "    \"verbose\": False,\n",
    "    \"warn\": True,\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Generating commonsense_qa...\n"
     ]
    }
   ],
   "source": [
    "#try commmonsense prompt 3\n",
    "\n",
    "gpt_4_hard.generate(config=config)\n",
    "gpt_4_hard.evaluate()\n",
    "gpt_4_hard.dump(\"prompt_3_cs.json\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Generating worldtree...\n"
     ]
    }
   ],
   "source": [
    "coll.generate(config=config)\n",
    "coll.evaluate()\n",
    "coll.dump(\"single_meta_experiment_prompt_3_worldtree.json\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/anaconda3/lib/python3.9/site-packages/scipy/__init__.py:146: UserWarning: A NumPy version >=1.16.5 and <1.23.0 is required for this version of SciPy (detected version 1.24.2\n",
      "  warnings.warn(f\"A NumPy version >={np_minversion} and <{np_maxversion}\"\n",
      "[nltk_data] Downloading package punkt to\n",
      "[nltk_data]     /Users/robertpraas/nltk_data...\n",
      "[nltk_data]   Package punkt is already up-to-date!\n"
     ]
    }
   ],
   "source": [
    "from cot import Collection\n",
    "gpt_4_hard = Collection.from_json(\"prompt_3_cs.json\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "config={\n",
    "    # We compare three different prompts for the chain of thought generation:\n",
    "    # \"Answer: Let's think step by step.\" and 'Answer: We should think about this step by step.', and \"Answer: First,\" \n",
    "    \"cot_trigger_keys\": ['praas-04'],\n",
    "\n",
    "    # We use the same answer extraction prompt for all three prompts\n",
    "    # \"Therefore, among A through D, the answer is\"\n",
    "    \"answer_extraction_keys\": ['auto-kojima'],#['kojima-A-D'], \n",
    "    \n",
    "    \"author\" : \"Robert\",\n",
    "    \"api_service\": \"openai_chat\", \n",
    "    \"engine\": \"gpt-4\", \n",
    "    \"temperature\": 0,\n",
    "    \"max_tokens\": 512,\n",
    "    \"verbose\": False,\n",
    "    \"warn\": True,\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Generating commonsense_qa...\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "b12efea06ac3490caf087e71980ec930",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/28 [00:00<?, ?ex/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Retrying langchain.chat_models.openai.ChatOpenAI.completion_with_retry.<locals>._completion_with_retry in 4.0 seconds as it raised APIError: Bad gateway. {\"error\":{\"code\":502,\"message\":\"Bad gateway.\",\"param\":null,\"type\":\"cf_bad_gateway\"}} 502 {'error': {'code': 502, 'message': 'Bad gateway.', 'param': None, 'type': 'cf_bad_gateway'}} {'Date': 'Mon, 17 Apr 2023 12:23:01 GMT', 'Content-Type': 'application/json', 'Content-Length': '84', 'Connection': 'keep-alive', 'X-Frame-Options': 'SAMEORIGIN', 'Referrer-Policy': 'same-origin', 'Cache-Control': 'private, max-age=0, no-store, no-cache, must-revalidate, post-check=0, pre-check=0', 'Expires': 'Thu, 01 Jan 1970 00:00:01 GMT', 'Server': 'cloudflare', 'CF-RAY': '7b949b0b8f3dc320-VIE'}.\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "facea596f88848f7aceb898c9c8d6c3b",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/28 [00:00<?, ?ex/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "fd1193be2c7b424ea27e2cbc0f0c6318",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Creating json from Arrow format:   0%|          | 0/1 [00:00<?, ?ba/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "gpt_4_hard.generate(config=config)\n",
    "gpt_4_hard.evaluate()\n",
    "gpt_4_hard.dump(\"prompt_4_cs.json\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [],
   "source": [
    "config={\n",
    "    # We compare three different prompts for the chain of thought generation:\n",
    "    # \"Answer: Let's think step by step.\" and 'Answer: We should think about this step by step.', and \"Answer: First,\" \n",
    "    \"cot_trigger_keys\": ['praas-05'],\n",
    "\n",
    "    # We use the same answer extraction prompt for all three prompts\n",
    "    # \"Therefore, among A through D, the answer is\"\n",
    "    \"answer_extraction_keys\": ['auto-kojima'],#['kojima-A-D'], \n",
    "    \n",
    "    \"author\" : \"Robert\",\n",
    "    \"api_service\": \"openai_chat\", \n",
    "    \"engine\": \"gpt-4\", \n",
    "    \"temperature\": 0,\n",
    "    \"max_tokens\": 512,\n",
    "    \"verbose\": False,\n",
    "    \"warn\": True,\n",
    "}\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Generating strategy_qa...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Retrying langchain.chat_models.openai.ChatOpenAI.completion_with_retry.<locals>._completion_with_retry in 4.0 seconds as it raised RateLimitError: That model is currently overloaded with other requests. You can retry your request, or contact us through our help center at help.openai.com if the error persists. (Please include the request ID 264bc179d64057c76b92a4864983b846 in your message.).\n",
      "Retrying langchain.chat_models.openai.ChatOpenAI.completion_with_retry.<locals>._completion_with_retry in 4.0 seconds as it raised RateLimitError: That model is currently overloaded with other requests. You can retry your request, or contact us through our help center at help.openai.com if the error persists. (Please include the request ID ef6cf4fe94dd7d65e4fe1bd99f365433 in your message.).\n"
     ]
    }
   ],
   "source": [
    "gpt_4_hard.generate(config=config)\n",
    "gpt_4_hard.evaluate()\n",
    "gpt_4_hard.dump(\"single_meta_experiment_prompt_5_new_data.json\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'open_book_qa': {'test': {'accuracy': {'gpt-4': {'None_praas-05_kojima-A-D': 0.0,\n",
       "     'None_zhou-01_kojima-A-D': 0.0}}}},\n",
       " 'strategy_qa': {'train': {'accuracy': {'gpt-4': {'None_zhou-01_kojima-yes-no': 0.0}}}}}"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gpt_4_hard.evaluate()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [],
   "source": [
    "gpt_4_hard.dump(\"single_meta_experiment_prompt_5_new_data.json\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [],
   "source": [
    "config={\n",
    "    # We compare three different prompts for the chain of thought generation:\n",
    "    # \"Answer: Let's think step by step.\" and 'Answer: We should think about this step by step.', and \"Answer: First,\" \n",
    "    \"cot_trigger_keys\": ['praas-06'],\n",
    "\n",
    "    # We use the same answer extraction prompt for all three prompts\n",
    "    # \"Therefore, among A through D, the answer is\"\n",
    "    \"answer_extraction_keys\": ['auto-kojima'],#['kojima-A-D'], \n",
    "    \n",
    "    \"author\" : \"Robert\",\n",
    "    \"api_service\": \"openai_chat\", \n",
    "    \"engine\": \"gpt-4\", \n",
    "    \"temperature\": 0,\n",
    "    \"max_tokens\": 512,\n",
    "    \"verbose\": False,\n",
    "    \"warn\": True,\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Generating strategy_qa...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Retrying langchain.chat_models.openai.ChatOpenAI.completion_with_retry.<locals>._completion_with_retry in 4.0 seconds as it raised APIError: Bad gateway. {\"error\":{\"code\":502,\"message\":\"Bad gateway.\",\"param\":null,\"type\":\"cf_bad_gateway\"}} 502 {'error': {'code': 502, 'message': 'Bad gateway.', 'param': None, 'type': 'cf_bad_gateway'}} {'Date': 'Fri, 21 Apr 2023 15:37:59 GMT', 'Content-Type': 'application/json', 'Content-Length': '84', 'Connection': 'keep-alive', 'X-Frame-Options': 'SAMEORIGIN', 'Referrer-Policy': 'same-origin', 'Cache-Control': 'private, max-age=0, no-store, no-cache, must-revalidate, post-check=0, pre-check=0', 'Expires': 'Thu, 01 Jan 1970 00:00:01 GMT', 'Server': 'cloudflare', 'CF-RAY': '7bb6ae894d5f0a30-ARN'}.\n",
      "Retrying langchain.chat_models.openai.ChatOpenAI.completion_with_retry.<locals>._completion_with_retry in 4.0 seconds as it raised APIError: Bad gateway. {\"error\":{\"code\":502,\"message\":\"Bad gateway.\",\"param\":null,\"type\":\"cf_bad_gateway\"}} 502 {'error': {'code': 502, 'message': 'Bad gateway.', 'param': None, 'type': 'cf_bad_gateway'}} {'Date': 'Fri, 21 Apr 2023 15:43:16 GMT', 'Content-Type': 'application/json', 'Content-Length': '84', 'Connection': 'keep-alive', 'X-Frame-Options': 'SAMEORIGIN', 'Referrer-Policy': 'same-origin', 'Cache-Control': 'private, max-age=0, no-store, no-cache, must-revalidate, post-check=0, pre-check=0', 'Expires': 'Thu, 01 Jan 1970 00:00:01 GMT', 'Server': 'cloudflare', 'CF-RAY': '7bb6b63c3f970a30-ARN'}.\n",
      "Retrying langchain.chat_models.openai.ChatOpenAI.completion_with_retry.<locals>._completion_with_retry in 4.0 seconds as it raised APIError: Bad gateway. {\"error\":{\"code\":502,\"message\":\"Bad gateway.\",\"param\":null,\"type\":\"cf_bad_gateway\"}} 502 {'error': {'code': 502, 'message': 'Bad gateway.', 'param': None, 'type': 'cf_bad_gateway'}} {'Date': 'Fri, 21 Apr 2023 15:50:32 GMT', 'Content-Type': 'application/json', 'Content-Length': '84', 'Connection': 'keep-alive', 'X-Frame-Options': 'SAMEORIGIN', 'Referrer-Policy': 'same-origin', 'Cache-Control': 'private, max-age=0, no-store, no-cache, must-revalidate, post-check=0, pre-check=0', 'Expires': 'Thu, 01 Jan 1970 00:00:01 GMT', 'Server': 'cloudflare', 'CF-RAY': '7bb6c0e3ef7c0a30-ARN'}.\n"
     ]
    }
   ],
   "source": [
    "gpt_4_hard.generate(config=config)\n",
    "gpt_4_hard.evaluate()\n",
    "gpt_4_hard.dump(\"single_meta_experiment_prompt_6.json\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'strategy_qa': {'train': {'accuracy': {'gpt-4': {'None_praas-05_kojima-yes-no': 0.2,\n",
       "     'None_praas-06_kojima-yes-no': 0.1,\n",
       "     'None_zhou-01_kojima-yes-no': 0.0}}}}}"
      ]
     },
     "execution_count": 41,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gpt_4_hard.evaluate()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\"Answer the question, then critique the answer. Based on the critique, reconsider the other answer options and give a single final answer. Answer: Let's work this out in a step by step way to be sure we have the right answer.\"\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "Answer: Let's work this out in a step by step way to be sure we have the right answer. Critique the initial answer, reconsider the other answer options and then give a single final answer."
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "base",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.12"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
